forked from mikeizbicki/twitter_postgres_parallel
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathload_tweets_parallel.sh
More file actions
executable file
·16 lines (14 loc) · 931 Bytes
/
load_tweets_parallel.sh
File metadata and controls
executable file
·16 lines (14 loc) · 931 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/sh
files=$(find data/*)
echo '================================================================================'
echo 'load pg_denormalized'
echo '================================================================================'
time echo "$files" | parallel ./load_denormalized.sh
echo '================================================================================'
echo 'load pg_normalized'
echo '================================================================================'
time echo "$files" | parallel python3 load_tweets.py --db "postgresql://postgres:pass@localhost:1318" --inputs
echo '================================================================================'
echo 'load pg_normalized_batch'
echo '================================================================================'
time echo "$files" | parallel python3 -u load_tweets_batch.py --db "postgresql://postgres:pass@localhost:1320" --inputs