trying to generate billions of individuals when some frequencies are 0

[samthiriot]

When working on a case with many empty cells in frequency, some parameters lead to the generation of very big populations.

To reproduce the case, using INSEE data:

library(data.table)
library(devtools)
load_all()

dwellings_raw <- read.csv(
        file="~/projets/2017\ parcimonious\ iterated\ picking/application_lille/FD_LOGEMTZB_2014.txt", 
        header=T, 
        nrow=50000, 
        sep=";",
        check.names=FALSE
        #,
        #col_types = cols(b=col_factor())
        )

# INPER: nb personnes ménage
# NBPI: nb pieces logement
# SURF: surface logement

sample_dwellings <- gosp.dpp::create_sample(
                data=dwellings_raw,
                encoding = list(
                        # we provide no mapping
                       ),
                weight.colname="IPONDL"
                )

# free some memory
remove(dwellings_raw)

#CATL: categorie
#   1 : Résidences principales
#   2 : Logements occasionnels
#   3 : Résidences secondaires
#   4 : Logements vacants
#   Z : Hors logement ordinaire
#
# n'y mettre un foyer que si vacant
pdi <- create_degree_probabilities_table(
                data.frame(
                    'CATL=1'=c(0.0, 1.0),
                    'CATL=2'=c(1.0, 0.0),
                    'CATL=3'=c(1.0, 0.0),
                    'CATL=4'=c(1.0, 0.0),
                    'CATL=Z'=c(1.0, 0.0),
                    check.names=FALSE
                    )
                )

#
households_raw <- read.csv(
        file="~/projets/2017\ parcimonious\ iterated\ picking/application_lille/FD_INDCVIZB_2014.txt", 
        header=T, 
        nrow=10000, 
        sep=";",
        check.names=FALSE
        #,
        #col_types = cols(b=col_factor())
        )
sample_households <- gosp.dpp::create_sample(
                data=households_raw,
                encoding = list(
                        # we provide no mapping
                       ),
                weight.colname="IPONDI"
                )
remove(households_raw)

pdj <- create_degree_probabilities_table(
                data.frame(
                    'STOCD=00'=c(1.0, 0.000001),
                    'STOCD=10'=c(0.000001, 1.0),
                    'STOCD=21'=c(0.000001, 1.0),
                    'STOCD=22'=c(0.000001, 1.0),
                    'STOCD=23'=c(0.000001, 1.0),
                    'STOCD=30'=c(1.0, 0.000001),
                    'STOCD=ZZ'=c(1.0, 0.000001),
                    check.names=FALSE
                    ),
                norm=TRUE
                )

# STOCD
    # 00 : Logement ordinaire inoccupé
    # 10 : Propriétaire
    # 21 : Locataire ou sous-locataire d'un logement loué vide non HLM
    # 22 : Locataire ou sous-locataire d'un logement loué vide HLM
    # 23 : Locataire ou sous-locataire d'un logement loué meublé ou d'une chambre d'hôtel
    # 30 : Logé gratuitement
    # ZZ : Hors logement ordinaire

# TYPL
    # Type de logement
    # 1 : Maison
    # 2 : Appartement
    # 3 : Logement-foyer
    # 4 : Chambre d'hôtel
    # 5 : Habitation de fortune
    # 6 : Pièce indépendante (ayant sa propre entrée)
    # Z : Hors logement ordinaire

# INPER: nb personnes ménage

# SURF: surface logement
    # 6 5 3 4 7 1 2

pij <- create_matching_probabilities_table(
        normalise(
            data.frame(
                "SURF=1"=c(1.0, 1.0, 0.7, 0.4, 0.1, 0.1, 0.001, 0.001, 0.001, 0.001, 0.001, 1.0), 
                "SURF=2"=c(1.0, 1.0, 1.0, 0.7, 0.4, 0.1, 0.1, 0.001, 0.001, 0.001, 0.001, 0.0), 
                "SURF=3"=c(0.8, 1.0, 1.0, 1.0, 0.7, 0.4, 0.1, 0.1, 0.001, 0.001, 0.001, 0.0), 
                "SURF=4"=c(0.3, 0.8, 1.0, 1.0, 1.0, 0.7, 0.4, 0.1, 0.1, 0.001, 0.001, 0.0), 
                "SURF=5"=c(0.3, 0.3, 0.8, 1.0, 1.0, 1.0, 0.7, 0.4, 0.1, 0.1, 0.1, 0.0), 
                "SURF=6"=c(0.1, 0.3, 0.3, 0.8, 1.0, 1.0, 1.0, 0.7, 0.4, 0.1, 0.1, 0.0), 
                "SURF=7"=c(0.01,  0.1, 0.3, 0.3, 0.8, 1.0, 1.0, 1.0, 0.7, 0.4, 0.4, 0.0), 
                row.names=c("INPER=1", "INPER=2", "INPER=3",  "INPER=4",  "INPER=5",  "INPER=6",  "INPER=7",  "INPER=8",  "INPER=9",  "INPER=10",  "INPER=11", "INPER=Z"), 
                check.names=FALSE
                )
            )
        )

prepared <- matching.prepare(sample_dwellings, sample_households, pdi, pdj, pij) 

solved <- matching.solve(prepared, nA=50000, nB=40000, nu.A=1, phi.A=0, delta.A=1, gamma=1, delta.B=1, phi.B=1, nu.B=1, verbose=T)

solved$gen$hat.nB
[1] 7002050000

[samthiriot]

as a first step, added a verification before freezing the machine

> case <- matching.generate(solved, sample_dwellings, sample_households, verbose=TRUE)
starting generation
Error in matching.generate(solved, sample_dwellings, sample_households,  : 
  you are trying to generate a population of more than 1 million of individuals: case$gen$hat.nA=50000, case$gen$hat.nB=7002050000 .
That probably does not make much sense. It also would likely fail and freeze your system.
If you really want to do it, add parameter force=TRUE and cross fingers.
> 

[samthiriot]

modified the computation of errors so the solutions with large counts of entities are not prefered in case another solution is available