Update growth_criteria() to use different translat...

Update growth_criteria() to use different translation for pop
/emp data
https://github.com/Transport-for-the-North/NorMITs-Demand/blob/b8df2cc936ecd6a3f746a695e21c7caa580ceeb7/external_forecast_system.py#L1151

    def _handle_growth_criteria(self,
                                synth_productions: pd.DataFrame,
                                synth_attractions: pd.DataFrame,
                                base_year: str,
                                future_years: List[str],
                                ) -> Tuple[pd.DataFrame, pd.DataFrame]:
        # Init
        all_years = [base_year] + future_years

        # Load the exceptional zone definitions from production/attraction
        # generation
        print("Loading Exceptional Growth Datafiles")
        exceptional_zones = eg.load_exceptional_zones(
            productions_export=self.exports["productions"],
            attractions_export=self.exports["attractions"]
        )
        # Reload aggregated population and employment data to calculate
        # sector level trip rates
        fname = consts.POP_FNAME % self.input_zone_system
        grown_pop_path = os.path.join(self.exports["productions"], fname)

        fname = consts.EMP_FNAME % self.input_zone_system
        grown_emp_path = os.path.join(self.exports["attractions"], fname)

        # For testing purposes - use the previously generated trip outputs -
        # same as the synthetic base
        obs_production_path = r"Y:\NorMITs Demand\norms_2015\v2_3-EFS_Output\iter1\Productions\norms_2015_productions.csv"
        obs_attraction_path = r"Y:\NorMITs Demand\norms_2015\v2_3-EFS_Output\iter1\Attractions\norms_2015_attractions.csv"

        # For testing purposes - use the converted productions/attractions
        # from a previous run (same as observed placeholders)
        # converted_productions = pd.read_csv(obs_production_path)
        # converted_pure_attractions = pd.read_csv(obs_attraction_path)

        # Detect the segment columns for PAs and Pop/Emp
        p_segs = du.list_safe_remove(list(synth_productions), all_years)
        a_segs = du.list_safe_remove(list(synth_attractions), all_years)

        growth_criteria_segments = {
            "pop": [seg for seg in p_segs if seg != "purpose_id"],
            "emp": [seg for seg in a_segs if seg != "purpose_id"] + ["employment_cat"],
            "prod": p_segs,
            "attr": a_segs
        }

        # ## APPLY GROWTH CRITERIA ## #
        # Placeholder sector file definition
        # TODO Integrate into EFS inputs
        model_zone_to_sector_path = r"Y:\NorMITs Demand\import\zone_translation\norms_2015_to_tfn_sectors.csv"
        from_zone_column = "norms_zone_id"
        to_sector_column = "tfn_sectors_zone_id"
        soc_weights_path = self.attraction_generator.imports["soc_weights"]

        # Load sector mapping for calculating the exceptional zone trip rates
        sector_lookup = pd.read_csv(model_zone_to_sector_path).rename(
            columns={
                from_zone_column: "model_zone_id",
                to_sector_column: "grouping_id"
            })
        sector_lookup = sector_lookup.set_index("model_zone_id")["grouping_id"]

        # Zone translation arguments for population/employment and
        # exceptional zone translation - reduces number of arguments required
        pop_translation, emp_translation = self.get_translation_dfs()

        # TODO: Update growth_criteria() to use different translation for pop
        #  /emp data
        zone_translator_args = {
            "translation_dataframe": pop_translation,
            "start_zoning_system_name": self.input_zone_system,
            "end_zoning_system_name": self.output_zone_system,
        }

        # MSOA path to translate population and employment zones
        msoa_lookup_path = os.path.join(
            self.imports["default_inputs"],
            self.msoa_lookup_path
        )

        # TODO: How to deal with NHB productions/attractions??
        #  Run this after the P/A models, then base the NHB off this?
        # Apply growth criteria to "normal" and "exceptional" zones
        productions, attractions = eg.growth_criteria(
            synth_productions=synth_productions,
            synth_attractions=synth_attractions,
            observed_prod_path=obs_production_path,
            observed_attr_path=obs_attraction_path,
            population_path=grown_pop_path,
            employment_path=grown_emp_path,
            msoa_lookup_path=msoa_lookup_path,
            segments=growth_criteria_segments,
            future_years=future_years,
            base_year=base_year,
            zone_translator=self.zone_translator,
            zone_translator_args=zone_translator_args,
            exceptional_zones=exceptional_zones,
            trip_rate_sectors=sector_lookup,
            soc_weights_path=soc_weights_path
        )

        return productions, attractions

    def get_translation_dfs(self) -> Tuple[pd.DataFrame, pd.DataFrame]:
        """
        Returns the dataframes for translating from input_zone_system to
        output_zone_system.

        A different translation dataframe is returned for translating
        population data and employment data.

        Returns
        -------
        pop_translation:
            A population weighted translation dataframe to convert from
            input_zone_system to output_zone_system

        emp_translation:
            A employment weighted translation dataframe to convert from
            input_zone_system to output_zone_system
        """
        # Init
        fname_args = (self.input_zone_system, self.output_zone_system)

        # Read in pop translation
        fname = consts.POP_TRANSLATION_FNAME % fname_args
        path = os.path.join(self.imports['zoning'], fname)
        pop_translation = pd.read_csv(path)

        # Read in emp translation
        fname = consts.EMP_TRANSLATION_FNAME % fname_args
        path = os.path.join(self.imports['zoning'], fname)
        emp_translation = pd.read_csv(path)

        return pop_translation, emp_translation

    def pa_to_od(self,
                 years_needed: List[int] = consts.ALL_YEARS,
                 modes_needed: List[int] = consts.MODES_NEEDED,
c0ee174f427fbc7a0c6b8b4a747048faa95256a6
Transport-for-the-North / NorMITs-Demand

Update growth_criteria() to use different translat... #661
