#!/bin/bash
#
# pack files like this:
# - no shard can exceed 85k files
# - no shard can exceed 84k*30k bytes
# (in other words we're expecting the avg size of a document to be 30k).
#
mkdir meta-split
mkdir xq-split
mkdir xq-log
rm -f meta-split/*
rm -f xq-log/*

echo "Partitioning files into different sets ..."
fpart -f 85193 -s $((85193 * 30000)) `pwd`/threads/ -o meta-split/files-shard

DB_NAME="linuxquestions"

# Plan out each shard contents
for f in $(find meta-split/ -type f); do
    if [[ ! -s "$f" ]]; then
        continue
    fi

    # extract partition number
    p=$(echo "$f" | perl -pne 's{^.*\.(\d+)$}{$1}g;')
    echo "$p -- $f"
    SHARD_IMPORT_SCRIPT="xq-split/shard-import.$p.xq"
 
    # generate import scripts
    cat << EOF > $SHARD_IMPORT_SCRIPT
DROP   DB $DB_NAME-shard-$p
CREATE DB $DB_NAME-shard-$p
SET PARSER html
SET HTMLPARSER method=xml,nons=true,nocdata=true,nodefaults=true,nobogons=true,nocolons=true,ignorable=true                                                                                                       
SET CREATEFILTER *.html
SET FTINDEX ON
SET TOKENINDEX ON
SET UPDINDEX TRUE
SET DEBUG TRUE
SET AUTOOPTIMIZE TRUE
EOF
    cat $f | perl -pne 's{^}{ADD }g;' >> $SHARD_IMPORT_SCRIPT

done

# Import data for each shard
echo "Import start"
for f in $(find xq-split/ -type f); do
    if [[ ! -s "$f" ]]; then
        continue
    fi
    p=$(echo "$f" | perl -pne 's{^.*\.(\d+)\.xq$}{$1}g;')
    /share/Public/builds/basex/bin/basexclient -U admin -P admin < $f | tee xq-log/shard-import.$p.log 2>&1 
done



