Merge pull request #232 from Baltic-RCC/dev

Dev to prod release

Author: VeikoAunapuu

Pipfile

@@ -6,7 +6,7 @@ name = "pypi"
 [packages]
 pandas = "2.2.2"
 numpy = "2.0.0"
-pypowsybl = "1.8.1"
+pypowsybl = "1.9.0"
 aniso8601 = "*"
 lxml = "*"
 edx = "*"

Pipfile.lock

@@ -6,15 +6,15 @@ FROM python:3.11 as emfos-model-retriever
 RUN pip install --upgrade setuptools
 RUN pip install --upgrade pip
 RUN pip install uv
-#RUN pip install pipenv
+RUN pip install pipenv
 
 # Install needed modules
 COPY Pipfile .
 COPY Pipfile.lock .
-COPY requirements.txt .
+#COPY requirements.txt .
 #RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
 #RUN pipenv install --system --deploy
-#RUN pipenv requirements > requirements.txt
+RUN pipenv requirements > requirements.txt
 RUN uv pip sync requirements.txt --system
 #RUN uv cache clean
 
@@ -42,15 +42,15 @@ FROM python:3.11 as emfos-model-merger
 RUN pip install --upgrade setuptools
 RUN pip install --upgrade pip
 RUN pip install uv
-#RUN pip install pipenv
+RUN pip install pipenv
 
 # Install needed modules
 COPY Pipfile .
 COPY Pipfile.lock .
-COPY requirements.txt .
+#COPY requirements.txt .
 #RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
 #RUN pipenv install --system --deploy
-#RUN pipenv requirements > requirements.txt
+RUN pipenv requirements > requirements.txt
 RUN uv pip sync requirements.txt --system
 #RUN uv cache clean
 

docker/Dockerfile

@@ -121,6 +121,8 @@ def send_to_elastic_bulk(index: str,
                                      data=(ndjson.dumps(json_message_list[batch:batch + batch_size])+"\n").encode(),
                                      timeout=None,
                                      headers={"Content-Type": "application/x-ndjson"})
+            if json.loads(response.content).get('errors'):
+                logger.error(f"Send to elasticsearch responded with errors: {response.text}")
             if debug:
                 logger.debug(f"ELK response: {response.content}")
             response_list.append(response.ok)
@@ -219,15 +221,12 @@ def __init__(self,
         self.session.auth = auth
 
     def handle(self, byte_string, properties):
-
-        Elastic.send_to_elastic_bulk(index=self.index,
-                                     json_message_list=json.loads(byte_string),
-                                     id_from_metadata=self.id_from_metadata,
-                                     id_metadata_list=self.id_metadata_list,
-                                     server=self.server,
-                                     debug=self.debug)
-
-        # TODO add support for properties argument
+        response = Elastic.send_to_elastic_bulk(index=self.index,
+                                                json_message_list=json.loads(byte_string),
+                                                id_from_metadata=self.id_from_metadata,
+                                                id_metadata_list=self.id_metadata_list,
+                                                server=self.server,
+                                                debug=self.debug)
 
 
 if __name__ == '__main__':

emf/common/integrations/elastic.py

@@ -234,10 +234,11 @@ def get_slack_generators(network: pypowsybl.network):
     return slack_generators
 
 
-def get_connected_components_data(network: pypowsybl.network, bus_count_threshold: int | None = None,
-                                  country_name: str = 'country'):
+def get_connected_components_data(network: pypowsybl.network,
+                                  bus_count_threshold: int | None = None,
+                                  country_col_name: str = 'country'):
     buses = get_network_elements(network, pypowsybl.network.ElementType.BUS)
-    data = buses.groupby('connected_component').agg(countries=(country_name, lambda x: list(x.unique())),
+    data = buses.groupby('connected_component').agg(countries=(country_col_name, lambda x: list(x.unique())),
                                                     bus_count=('name', 'size'))
     if bus_count_threshold:
         data = data[data.bus_count > bus_count_threshold]

emf/loadflow_tool/helper.py

@@ -119,7 +119,8 @@
 }
 
 __CGM_RELAXED_2_PROVIDER = {
-    'loadPowerFactorConstant': 'False',  # cim:PowerFlowSettings.loadVoltageDependency "false" ; TODO - check this
+    # 'loadPowerFactorConstant': 'False',  # cim:PowerFlowSettings.loadVoltageDependency "false" ; TODO - check this
+    'loadPowerFactorConstant': 'True',
     'maxOuterLoopIterations': '30',  # eumd:PowerFlowSettings.maxIterationNumber "30"
     'lowImpedanceThreshold': '1.0E-5',  # cim:PowerFlowSettings.impedanceThreshold "1e-05" ;
     'newtonRaphsonStoppingCriteriaType': 'PER_EQUATION_TYPE_CRITERIA',
@@ -214,7 +215,8 @@
     read_slack_bus=True,
     write_slack_bus=False,
     distributed_slack=True,  # cim:PowerFlowSettings.slackDistributionKind cim:SlackDistributionKind.generationDistributionActivePowerAndVoltageNodesOnly ;
-    balance_type=pypowsybl._pypowsybl.BalanceType.PROPORTIONAL_TO_GENERATION_REMAINING_MARGIN, #cim:PowerFlowSettings.slackDistributionKind cim:SlackDistributionKind.generationDistributionActivePowerAndVoltageNodesOnly ;
+    # balance_type=pypowsybl._pypowsybl.BalanceType.PROPORTIONAL_TO_GENERATION_REMAINING_MARGIN, #cim:PowerFlowSettings.slackDistributionKind cim:SlackDistributionKind.generationDistributionActivePowerAndVoltageNodesOnly ;
+    balance_type=pypowsybl._pypowsybl.BalanceType.PROPORTIONAL_TO_CONFORM_LOAD,
     dc_use_transformer_ratio=None,
     countries_to_balance=None,
     connected_component_mode=pypowsybl._pypowsybl.ConnectedComponentMode.ALL,

emf/loadflow_tool/loadflow_settings.py

@@ -248,13 +248,28 @@ def handle(self, task_object: dict, **kwargs):
                                 .groupby('id').agg(curve_p_min=('p', 'min'), curve_p_max=('p', 'max'))).reset_index()
                 curve_generators = generators.merge(curve_limits, on='id')
                 # low end can be zero
-                curve_generators = curve_generators[(curve_generators['target_p'] > curve_generators['curve_p_max'])]
+                curve_generators = curve_generators[(curve_generators['target_p'] > curve_generators['curve_p_max']) |
+                                                    ((curve_generators['target_p'] > 0) &
+                                                     (curve_generators['target_p'] < curve_generators['curve_p_min']))]
                 if not curve_generators.empty:
                     logger.warning(f"Found {len(curve_generators.index)} generators for "
-                                   f"which p > max(reactive capacity curve(p))")
-                    # Solution 1: set max_p from curve max, it should contain p on target_p
-                    # curve_generators['max_p'] = curve_generators['curve_p_max']
-                    # network_pre_instance.update_generators(curve_generators[['id', 'max_p']].set_index('id'))
+                                   f"which p > max(reactive capacity curve(p)) or p < min(reactive capacity curve(p))")
+                    # in order this to work curve_p_min <= p_min <= target_p <= p_max <= curve_p_max
+                    # Solution 1: set max_p from curve max, it should contain p on target-p
+                    upper_limit_violated = curve_generators[
+                        (curve_generators['max_p'] > curve_generators['curve_p_max'])]
+                    if not upper_limit_violated.empty:
+                        logger.warning(f"Updating max p from curve for {len(upper_limit_violated.index)} generators")
+                        upper_limit_violated['max_p'] = upper_limit_violated['curve_p_max']
+                        network_pre_instance.update_generators(upper_limit_violated[['id', 'max_p']].set_index('id'))
+
+                    lower_limit_violated = curve_generators[
+                        (curve_generators['min_p'] < curve_generators['curve_p_min'])]
+                    if not lower_limit_violated.empty:
+                        logger.warning(f"Updating min p from curve for {len(lower_limit_violated.index)} generators")
+                        lower_limit_violated['min_p'] = lower_limit_violated['curve_p_min']
+                        network_pre_instance.update_generators(lower_limit_violated[['id', 'min_p']].set_index('id'))
+
                     # Solution 2: discard generator from participating
                     extensions = network_pre_instance.get_extensions('activePowerControl')
                     remove_curve_generators = extensions.merge(curve_generators[['id']],

emf/loadflow_tool/model_merger/model_merger.py

@@ -8,9 +8,8 @@
                                                             fix_sv_tapsteps, remove_duplicate_sv_voltages,
                                                             remove_small_islands,check_and_fix_dependencies,
                                                             disconnect_equipment_if_flow_sum_not_zero,
-                                                            export_to_cgmes_zip, set_brell_lines_to_zero_in_models,
-                                                            configure_paired_boundarypoint_injections_by_nodes,
-                                                            set_brell_lines_to_zero_in_models_new)
+                                                            export_to_cgmes_zip,
+                                                            configure_paired_boundarypoint_injections_by_nodes)
 
 
 logger = logging.getLogger(__name__)
@@ -19,17 +18,7 @@
 def run_pre_merge_processing(input_models, merging_area):
 
     # TODO warning logs for temp fix functions
-
-    # SET BRELL LINE VALUES
-    if merging_area == 'BA':
-        input_models = set_brell_lines_to_zero_in_models(input_models)
-
     assembled_data = load_opdm_data(input_models)
-
-    # TODO try to optimize it better
-    # if merging_area == 'BA':
-    #     assembled_data = set_brell_lines_to_zero_in_models_new(assembled_data)
-
     assembled_data = triplets.cgmes_tools.update_FullModel_from_filename(assembled_data)
     assembled_data = configure_paired_boundarypoint_injections_by_nodes(assembled_data)
     escape_upper_xml = assembled_data[assembled_data['VALUE'].astype(str).str.contains('.XML')]