29 lines
651 B
Bash
Executable File
29 lines
651 B
Bash
Executable File
#!/bin/bash
|
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
|
DATE=$1
|
|
|
|
SRC_ROOT=$DIR/../../
|
|
RAW_DATA=$SRC_ROOT/analysis_data/raw/$DATE
|
|
PROCESSED_DATA=$SRC_ROOT/analysis_data/processed/$DATE
|
|
|
|
mkdir -p $PROCESSED_DATA
|
|
rm $PROCESSED_DATA/* >/dev/null
|
|
|
|
ALL_COMMANDS=""
|
|
for IN_PATH in $RAW_DATA/*.gz
|
|
do
|
|
BASE_NAME=`basename $IN_PATH`
|
|
BASE_NAME_STEM=${BASE_NAME%.*}
|
|
for CMD in trades orders msgs books
|
|
do
|
|
OUT_PATH=$PROCESSED_DATA/$BASE_NAME_STEM.$CMD.gz
|
|
ALL_COMMANDS+="(cd $SPARK_BIN_ROOT && zcat $IN_PATH | ./sparkd parsehistoricalfix $CMD | gzip >$OUT_PATH)\n"
|
|
done
|
|
done
|
|
echo -e $ALL_COMMANDS | parallel --will-cite
|
|
|
|
|
|
|
|
|
|
|