#!/bin/bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" DATE=$1 SRC_ROOT=$DIR/../../ RAW_DATA=$SRC_ROOT/analysis_data/raw/$DATE PROCESSED_DATA=$SRC_ROOT/analysis_data/processed/$DATE mkdir -p $PROCESSED_DATA rm $PROCESSED_DATA/* >/dev/null ALL_COMMANDS="" for IN_PATH in $RAW_DATA/*.gz do BASE_NAME=`basename $IN_PATH` BASE_NAME_STEM=${BASE_NAME%.*} for CMD in trades orders msgs books do OUT_PATH=$PROCESSED_DATA/$BASE_NAME_STEM.$CMD.gz ALL_COMMANDS+="(cd $SPARK_BIN_ROOT && zcat $IN_PATH | ./sparkd parsehistoricalfix $CMD | gzip >$OUT_PATH)\n" done done echo -e $ALL_COMMANDS | parallel --will-cite