Files
spark/src/analysis/parse.sh
T
2020-02-03 15:48:20 -05:00

29 lines
651 B
Bash
Executable File

#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DATE=$1
SRC_ROOT=$DIR/../../
RAW_DATA=$SRC_ROOT/analysis_data/raw/$DATE
PROCESSED_DATA=$SRC_ROOT/analysis_data/processed/$DATE
mkdir -p $PROCESSED_DATA
rm $PROCESSED_DATA/* >/dev/null
ALL_COMMANDS=""
for IN_PATH in $RAW_DATA/*.gz
do
BASE_NAME=`basename $IN_PATH`
BASE_NAME_STEM=${BASE_NAME%.*}
for CMD in trades orders msgs books
do
OUT_PATH=$PROCESSED_DATA/$BASE_NAME_STEM.$CMD.gz
ALL_COMMANDS+="(cd $SPARK_BIN_ROOT && zcat $IN_PATH | ./sparkd parsehistoricalfix $CMD | gzip >$OUT_PATH)\n"
done
done
echo -e $ALL_COMMANDS | parallel --will-cite