add autoscaling

Author: Louis-Jellyfish

deployment/deploy_cloud.sh

@@ -41,6 +41,8 @@ python3 -m main \
     --template_location "gs://$2/templates/megalista" \
     --num_workers 1 \
     --autoscaling_algorithm=NONE \
+    --autoscaling_algorithm=THROUGHPUT_BASED \
+    --number_of_worker_harness_threads=16 \
     --service_account_email "$4" \
     --collect_usage_stats "$5"
 echo "Copy megalista_medata to bucket $2"

megalista_dataflow/steps/processing_steps.py

@@ -523,6 +523,8 @@ def expand(self, executions):
                 20,
                 TransactionalType.UUID,
             )
+            | "Prevent Fusion"
+            >> beam.Reshuffle()
             | "Upload - GA 4 measurement protocol"
             >> beam.ParDo(
                 GoogleAnalytics4MeasurementProtocolUploaderDoFn(