digital-guard / preserv-BR

Digital Preservation of Brazilian metadata of donated maps.
http://git.digital-guard.org/preserv-BR
3 stars 1 forks source link

Bage/_pk0082.01 make #99

Closed crebollobr closed 2 years ago

crebollobr commented 2 years ago

Erro no comando make via


carlos@addressforall:/var/gits/_dg/preserv-BR/data/RS/Bage/_pk0082.01$ make via pg_db=ingest20

------------------------------------------
------ BR-RS-Bage  ------
------ Layer tipo via_full  ------
-- Incluindo dados do arquivo-1 do package-7600008201101 na base ingest20 --
 Nome-hash do arquivo-1: 5816997b063b62b3d3d15ba8339a05ab1bbac651b645fddbea194d4b8a445932.zip
 Tabela do layer: pk7600008201101_p1_via
 Sub-arquivos do arquivo-1 com o conteúdo alvo: *LOGRADOUROS*
 Tema dos sub-arquivos: LOGRADOUROS
Run with tmux and sudo! (DANGER: seems not idempotent on psql)
carlos
Above user is root? If not, you have permissions for all paths?
[Press ENTER to continue or Ctrl+C to quit]
psql postgres://postgres@localhost/ingest20 -c "DROP TABLE IF EXISTS pk7600008201101_p1_via CASCADE"
NOTICE:  drop cascades to view vw1_pk7600008201101_p1_via
DROP TABLE
Extracting ....
cd /tmp/sandbox/_pkBR821_001; 7z  x -y /var/www/preserv.addressforall.org/download/5816997b063b62b3d3d15ba8339a05ab1bbac651b645fddbea194d4b8a445932.zip "*LOGRADOUROS*" ; chmod -R a+rwx . > /dev/null

7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
p7zip Version 16.02 (locale=pt_BR.UTF-8,Utf16=on,HugeFiles=on,64 bits,4 CPUs DO-Regular (306F2),ASM,AES-NI)

Scanning the drive for archives:
1 file, 456311 bytes (446 KiB)                        

Extracting archive: /var/www/preserv.addressforall.org/download/5816997b063b62b3d3d15ba8339a05ab1bbac651b645fddbea194d4b8a445932.zip
--
Path = /var/www/preserv.addressforall.org/download/5816997b063b62b3d3d15ba8339a05ab1bbac651b645fddbea194d4b8a445932.zip
Type = zip
Physical Size = 456311

Everything is Ok

Size:       1925924
Compressed: 456311
Conferindo se SRID 31981 está configurado:
 srid  |                                    proj4text                                     
-------+----------------------------------------------------------------------------------
 31981 | +proj=utm +zone=21 +south +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs 
(1 row)

[Press ENTER to continue or Ctrl+C to quit]
Executando geojson2sql ...
psql postgres://postgres@localhost/ingest20 -c "DROP TABLE IF EXISTS pk7600008201101_p1_via"
NOTICE:  table "pk7600008201101_p1_via" does not exist, skipping
DROP TABLE
psql postgres://postgres@localhost/ingest20 -c "CREATE TABLE pk7600008201101_p1_via AS SELECT * FROM geojson_readfile_features_jgeom('/tmp/sandbox/_pkBR821_001/LOGRADOUROS.geojson',null)"
SELECT 4383
psql postgres://postgres@localhost/ingest20 -c "CREATE VIEW vw1_pk7600008201101_p1_via AS SELECT row_number() OVER () as gid, properties->'logradouro' AS via_name, \
    jgeom AS geom \
    FROM pk7600008201101_p1_via"
CREATE VIEW
psql postgres://postgres@localhost/ingest20 -c "SELECT ingest.any_load('geojson2sql','/tmp/sandbox/_pkBR821_001/LOGRADOUROS.geojson','via_full','vw1_pk7600008201101_p1_via','7600008201101','5816997b063b62b3d3d15ba8339a05ab1bbac651b645fddbea194d4b8a445932.zip',array[]::text[],5,1)"
ERROR:  Geohash requires inputs in decimal degrees, got (776357 6.53117e+06, 776357 6.53117e+06).
CONTEXT:  SQL statement "
      WITH
      scan AS (
        SELECT 21 AS file_id,
                CASE
                 WHEN true THEN gid
                 ELSE gid+COALESCE((SELECT MAX(feature_id) FROM ingest.feature_asis WHERE file_id=21),0)
               END AS gid,
        properties,
               CASE
                 WHEN ST_SRID(geom)=0 THEN ST_SetSRID(geom,4326)
                 WHEN true AND ST_SRID(geom)!=4326 AND true THEN ST_Transform(geom,4326)
                 ELSE geom
               END AS geom
        FROM (
            SELECT gid,  -- feature_id_col
                 to_jsonb(subq) as properties,
                 ST_GeomFromGeoJSON(geom) AS geom -- geom
            FROM vw1_pk7600008201101_p1_via , LATERAL (SELECT via_name) subq
          ) t
      ),
      a0 AS (
        SELECT *
        FROM scan
        WHERE ST_IsValid(geom)
      ),
      a AS (
        SELECT *
        FROM a0
        WHERE ST_IsClosed(geom) = TRUE OR GeometryType(geom) IN ('LINESTRING','MULTILINESTRING')
      ),
      mask AS (SELECT ingest.buffer_geom(geom,1) AS geom FROM ingest.vw02full_donated_packfilevers WHERE id=7600008201101 LIMIT 1),
      b AS (
        SELECT file_id, gid, properties, geom, ( B'000000000' ||  (NOT(ST_IsSimple(geom)))::int::bit || (NOT(ST_IsValid(geom)))::int::bit || (NOT(ST_Intersects(geom,(SELECT geom FROM mask))))::int::bit ) AS error_mask
        FROM a
      ),
      c AS (
        (
        SELECT file_id, gid, properties, geom, (error_mask | (
            B'00' ||
            (CASE (SELECT (ingest.donated_PackComponent_geomtype(21))[1]) WHEN 'poly' THEN ST_Area(geom,true) > 2147483647 WHEN 'line' THEN ST_Length(geom,true) > 2147483647 ELSE FALSE END)::int::bit ||
            B'00' ||
            (GeometryType(geom) NOT IN ('LINESTRING','MULTILINESTRING'))::int::bit ||
            (geom IS NULL)::int::bit ||
            (CASE (SELECT (ingest.donated_PackComponent_geomtype(21))[1]) WHEN 'poly' THEN ST_Area(geom,true) < 5 WHEN 'line' THEN ST_Length(geom,true) < 2 ELSE FALSE END)::int::bit ||
            (ST_IsEmpty(geom))::int::bit ||
            B'000' )) AS error_mask
        FROM (
           SELECT file_id, gid,
                  properties,
                  CASE (SELECT (ingest.donated_PackComponent_geomtype(21))[1])
                    WHEN 'point' THEN ST_ReducePrecision( ST_Intersection(geom,(SELECT geom FROM mask)), 0.000001 )
                    ELSE ST_SimplifyPreserveTopology( -- remove collinear points 
                            ST_ReducePrecision( -- round decimal degrees of SRID 4326, ~1 meter
                              ST_Intersection( geom, (SELECT geom FROM mask) )
                              ,0.000001
                            ),
                            0.00000001
                    )
                  END AS geom,
                  error_mask
           FROM b
           WHERE bit_count(error_mask) = 0 
           ) t
        )
        UNION
        (
            SELECT * FROM b WHERE bit_count(error_mask) <> 0
        )
        UNION
        (
            SELECT file_id, gid, properties, ST_MakeValid(geom) AS geom, B'000100000000' AS error_mask
            FROM a0
            WHERE ST_IsClosed(geom) = FALSE AND GeometryType(geom) NOT IN ('LINESTRING','MULTILINESTRING')
        )
        UNION
        (
            SELECT file_id, gid, properties, ST_MakeValid(geom) AS geom, B'000000000010' AS error_mask
            FROM scan
            WHERE ST_IsValid(geom) = FALSE
        )
      ),
      stats AS (
      SELECT ARRAY [
            (SELECT COUNT(*) FROM scan)::bigint,
            (COUNT(*) filter (WHERE get_bit(error_mask,11) = 1))::bigint, -- intersects
            (COUNT(*) filter (WHERE get_bit(error_mask,10) = 1))::bigint, -- invalid
            (COUNT(*) filter (WHERE get_bit(error_mask, 9) = 1))::bigint, -- simple
            (COUNT(*) filter (WHERE get_bit(error_mask, 8) = 1))::bigint, -- empty
            (COUNT(*) filter (WHERE get_bit(error_mask, 7) = 1))::bigint, -- small
            (COUNT(*) filter (WHERE get_bit(error_mask, 6) = 1))::bigint, -- null
            (COUNT(*) filter (WHERE get_bit(error_mask, 5) = 1))::bigint, -- invalid_type
                                                                          -- bit 4 é reservado para duplicados
            (COUNT(*) filter (WHERE get_bit(error_mask, 3) = 1))::bigint, -- is_closed
            (COUNT(*) filter (WHERE get_bit(error_mask, 2) = 1))::bigint  -- large
                                                                          -- bit 1 uso futuro
                                                                          -- bit 0 uso futuro
            ]
        FROM c
      ),
      ins_asis AS (
        INSERT INTO ingest.feature_asis
        SELECT file_id, gid, properties, geom
        FROM c
            WHERE  bit_count(error_mask) = 0
        RETURNING 1
      ),
      ins_asis_discarded AS (
        INSERT INTO ingest.feature_asis_discarded
        SELECT file_id, gid, properties || jsonb_build_object('error_mask',error_mask), geom
        FROM c
            WHERE  bit_count(error_mask) <> 0 
        RETURNING 1
      )
      SELECT array_append(array_append( (SELECT * FROM stats), (SELECT COUNT(*) FROM ins_asis) ), (SELECT COUNT(*) FROM ins_asis_discarded))
    "
PL/pgSQL function ingest.any_load(text,text,text,text,bigint,text,text[],integer,integer,boolean,text,boolean) line 213 at EXECUTE
make: *** [makefile:245: via] Error 1
crebollobr commented 2 years ago

Erro no make block


carlos@addressforall:/var/gits/_dg/preserv-BR/data/RS/Bage/_pk0082.01$ make block pg_db=ingest20

------------------------------------------
------ BR-RS-Bage  ------
------ Layer tipo block_none  ------
-- Incluindo dados do arquivo-3 do package-7600008201301 na base ingest20 --
 Nome-hash do arquivo-3: 47063f994f552e27c3b528fcc60cb03e36923b6875dd862dc54062e0ecb3bf60.zip
 Tabela do layer: pk7600008201301_p3_block
 Sub-arquivos do arquivo-3 com o conteúdo alvo: *QUARTEIRAO*
 Tema dos sub-arquivos: quarteirões
Run with tmux and sudo! (DANGER: seems not idempotent on psql)
carlos
Above user is root? If not, you have permissions for all paths?
[Press ENTER to continue or Ctrl+C to quit]
psql postgres://postgres@localhost/ingest20 -c "DROP TABLE IF EXISTS pk7600008201301_p3_block CASCADE"
DROP TABLE
Extracting ....
cd /tmp/sandbox/_pkBR821_001; 7z  x -y /var/www/preserv.addressforall.org/download/47063f994f552e27c3b528fcc60cb03e36923b6875dd862dc54062e0ecb3bf60.zip "*QUARTEIRAO*" ; chmod -R a+rwx . > /dev/null

7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
p7zip Version 16.02 (locale=pt_BR.UTF-8,Utf16=on,HugeFiles=on,64 bits,4 CPUs DO-Regular (306F2),ASM,AES-NI)

Scanning the drive for archives:
1 file, 1496124 bytes (1462 KiB)                      

Extracting archive: /var/www/preserv.addressforall.org/download/47063f994f552e27c3b528fcc60cb03e36923b6875dd862dc54062e0ecb3bf60.zip
--
Path = /var/www/preserv.addressforall.org/download/47063f994f552e27c3b528fcc60cb03e36923b6875dd862dc54062e0ecb3bf60.zip
Type = zip
Physical Size = 1496124

Everything is Ok

Size:       4019215
Compressed: 1496124
Conferindo se SRID 31981 está configurado:
 srid  |                                    proj4text                                     
-------+----------------------------------------------------------------------------------
 31981 | +proj=utm +zone=21 +south +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs 
(1 row)

[Press ENTER to continue or Ctrl+C to quit]
Executando geojson2sql ...
psql postgres://postgres@localhost/ingest20 -c "DROP TABLE IF EXISTS pk7600008201301_p3_block"
NOTICE:  table "pk7600008201301_p3_block" does not exist, skipping
DROP TABLE
psql postgres://postgres@localhost/ingest20 -c "CREATE TABLE pk7600008201301_p3_block AS SELECT * FROM geojson_readfile_features_jgeom('/tmp/sandbox/_pkBR821_001/QUARTEIRAO.geojson',null)"
SELECT 2141
psql postgres://postgres@localhost/ingest20 -c "CREATE VIEW vw3_pk7600008201301_p3_block AS SELECT row_number() OVER () as gid, jgeom AS geom \
FROM pk7600008201301_p3_block"
CREATE VIEW
psql postgres://postgres@localhost/ingest20 -c "SELECT ingest.any_load('geojson2sql','/tmp/sandbox/_pkBR821_001/QUARTEIRAO.geojson','block_none','vw3_pk7600008201301_p3_block','7600008201301','47063f994f552e27c3b528fcc60cb03e36923b6875dd862dc54062e0ecb3bf60.zip',array[]::text[],5,1)"
ERROR:  Geohash requires inputs in decimal degrees, got (779590 6.52928e+06, 779590 6.52928e+06).
CONTEXT:  SQL statement "
      WITH
      scan AS (
        SELECT 22 AS file_id,
                CASE
                 WHEN true THEN gid
                 ELSE gid+COALESCE((SELECT MAX(feature_id) FROM ingest.feature_asis WHERE file_id=22),0)
               END AS gid,
        properties,
               CASE
                 WHEN ST_SRID(geom)=0 THEN ST_SetSRID(geom,4326)
                 WHEN true AND ST_SRID(geom)!=4326 AND true THEN ST_Transform(geom,4326)
                 ELSE geom
               END AS geom
        FROM (
            SELECT gid,  -- feature_id_col
                 to_jsonb(subq) as properties,
                 ST_GeomFromGeoJSON(geom) AS geom -- geom
            FROM vw3_pk7600008201301_p3_block , LATERAL (SELECT ) subq
          ) t
      ),
      a0 AS (
        SELECT *
        FROM scan
        WHERE ST_IsValid(geom)
      ),
      a AS (
        SELECT *
        FROM a0
        WHERE ST_IsClosed(geom) = TRUE OR GeometryType(geom) IN ('LINESTRING','MULTILINESTRING')
      ),
      mask AS (SELECT ingest.buffer_geom(geom,1) AS geom FROM ingest.vw02full_donated_packfilevers WHERE id=7600008201301 LIMIT 1),
      b AS (
        SELECT file_id, gid, properties, geom, ( B'000000000' ||  (NOT(ST_IsSimple(geom)))::int::bit || (NOT(ST_IsValid(geom)))::int::bit || (NOT(ST_Intersects(geom,(SELECT geom FROM mask))))::int::bit ) AS error_mask
        FROM a
      ),
      c AS (
        (
        SELECT file_id, gid, properties, geom, (error_mask | (
            B'00' ||
            (CASE (SELECT (ingest.donated_PackComponent_geomtype(22))[1]) WHEN 'poly' THEN ST_Area(geom,true) > 2147483647 WHEN 'line' THEN ST_Length(geom,true) > 2147483647 ELSE FALSE END)::int::bit ||
            B'00' ||
            (GeometryType(geom) NOT IN ('POLYGON'   ,'MULTIPOLYGON'))::int::bit ||
            (geom IS NULL)::int::bit ||
            (CASE (SELECT (ingest.donated_PackComponent_geomtype(22))[1]) WHEN 'poly' THEN ST_Area(geom,true) < 5 WHEN 'line' THEN ST_Length(geom,true) < 2 ELSE FALSE END)::int::bit ||
            (ST_IsEmpty(geom))::int::bit ||
            B'000' )) AS error_mask
        FROM (
           SELECT file_id, gid,
                  properties,
                  CASE (SELECT (ingest.donated_PackComponent_geomtype(22))[1])
                    WHEN 'point' THEN ST_ReducePrecision( ST_Intersection(geom,(SELECT geom FROM mask)), 0.000001 )
                    ELSE ST_SimplifyPreserveTopology( -- remove collinear points 
                            ST_ReducePrecision( -- round decimal degrees of SRID 4326, ~1 meter
                              ST_Intersection( geom, (SELECT geom FROM mask) )
                              ,0.000001
                            ),
                            0.00000001
                    )
                  END AS geom,
                  error_mask
           FROM b
           WHERE bit_count(error_mask) = 0 
           ) t
        )
        UNION
        (
            SELECT * FROM b WHERE bit_count(error_mask) <> 0
        )
        UNION
        (
            SELECT file_id, gid, properties, ST_MakeValid(geom) AS geom, B'000100000000' AS error_mask
            FROM a0
            WHERE ST_IsClosed(geom) = FALSE AND GeometryType(geom) NOT IN ('LINESTRING','MULTILINESTRING')
        )
        UNION
        (
            SELECT file_id, gid, properties, ST_MakeValid(geom) AS geom, B'000000000010' AS error_mask
            FROM scan
            WHERE ST_IsValid(geom) = FALSE
        )
      ),
      stats AS (
      SELECT ARRAY [
            (SELECT COUNT(*) FROM scan)::bigint,
            (COUNT(*) filter (WHERE get_bit(error_mask,11) = 1))::bigint, -- intersects
            (COUNT(*) filter (WHERE get_bit(error_mask,10) = 1))::bigint, -- invalid
            (COUNT(*) filter (WHERE get_bit(error_mask, 9) = 1))::bigint, -- simple
            (COUNT(*) filter (WHERE get_bit(error_mask, 8) = 1))::bigint, -- empty
            (COUNT(*) filter (WHERE get_bit(error_mask, 7) = 1))::bigint, -- small
            (COUNT(*) filter (WHERE get_bit(error_mask, 6) = 1))::bigint, -- null
            (COUNT(*) filter (WHERE get_bit(error_mask, 5) = 1))::bigint, -- invalid_type
                                                                          -- bit 4 é reservado para duplicados
            (COUNT(*) filter (WHERE get_bit(error_mask, 3) = 1))::bigint, -- is_closed
            (COUNT(*) filter (WHERE get_bit(error_mask, 2) = 1))::bigint  -- large
                                                                          -- bit 1 uso futuro
                                                                          -- bit 0 uso futuro
            ]
        FROM c
      ),
      ins_asis AS (
        INSERT INTO ingest.feature_asis
        SELECT file_id, gid, properties, geom
        FROM c
            WHERE  bit_count(error_mask) = 0
        RETURNING 1
      ),
      ins_asis_discarded AS (
        INSERT INTO ingest.feature_asis_discarded
        SELECT file_id, gid, properties || jsonb_build_object('error_mask',error_mask), geom
        FROM c
            WHERE  bit_count(error_mask) <> 0 
        RETURNING 1
      )
      SELECT array_append(array_append( (SELECT * FROM stats), (SELECT COUNT(*) FROM ins_asis) ), (SELECT COUNT(*) FROM ins_asis_discarded))
    "
PL/pgSQL function ingest.any_load(text,text,text,text,bigint,text,text[],integer,integer,boolean,text,boolean) line 213 at EXECUTE
make: *** [makefile:66: block] Error 1
crebollobr commented 2 years ago
carlos@addressforall:/var/gits/_dg/preserv-BR/data/RS/Bage/_pk0082.01$ make parcel pg_db=ingest20

------------------------------------------
------ BR-RS-Bage  ------
------ Layer tipo parcel_full  ------
-- Incluindo dados do arquivo-2 do package-7600008201201 na base ingest20 --
 Nome-hash do arquivo-2: 7fc034ac5c703dcfe93567bd5196ce6eba22df7b779b15ca713b026e85f64759.zip
 Tabela do layer: pk7600008201201_p2_parcel
 Sub-arquivos do arquivo-2 com o conteúdo alvo: *LOTES_URBANOS*
 Tema dos sub-arquivos: LOTES
Run with tmux and sudo! (DANGER: seems not idempotent on psql)
carlos
Above user is root? If not, you have permissions for all paths?
[Press ENTER to continue or Ctrl+C to quit]
psql postgres://postgres@localhost/ingest20 -c "DROP TABLE IF EXISTS pk7600008201201_p2_parcel CASCADE"
DROP TABLE
Extracting ....
cd /tmp/sandbox/_pkBR821_001; 7z  x -y /var/www/preserv.addressforall.org/download/7fc034ac5c703dcfe93567bd5196ce6eba22df7b779b15ca713b026e85f64759.zip "*LOTES_URBANOS*" ; chmod -R a+rwx . > /dev/null

7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
p7zip Version 16.02 (locale=pt_BR.UTF-8,Utf16=on,HugeFiles=on,64 bits,4 CPUs DO-Regular (306F2),ASM,AES-NI)

Scanning the drive for archives:
1 file, 10319163 bytes (10078 KiB)                    

Extracting archive: /var/www/preserv.addressforall.org/download/7fc034ac5c703dcfe93567bd5196ce6eba22df7b779b15ca713b026e85f64759.zip
--
Path = /var/www/preserv.addressforall.org/download/7fc034ac5c703dcfe93567bd5196ce6eba22df7b779b15ca713b026e85f64759.zip
Type = zip
Physical Size = 10319163

Everything is Ok            

Size:       39988323
Compressed: 10319163
Conferindo se SRID 31981 está configurado:
 srid  |                                    proj4text                                     
-------+----------------------------------------------------------------------------------
 31981 | +proj=utm +zone=21 +south +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs 
(1 row)

[Press ENTER to continue or Ctrl+C to quit]
Executando geojson2sql ...
psql postgres://postgres@localhost/ingest20 -c "DROP TABLE IF EXISTS pk7600008201201_p2_parcel"
NOTICE:  table "pk7600008201201_p2_parcel" does not exist, skipping
DROP TABLE
psql postgres://postgres@localhost/ingest20 -c "CREATE TABLE pk7600008201201_p2_parcel AS SELECT * FROM geojson_readfile_features_jgeom('/tmp/sandbox/_pkBR821_001/LOTES_URBANOS.geojson',null)"
SELECT 42726
psql postgres://postgres@localhost/ingest20 -c "CREATE VIEW vw2_pk7600008201201_p2_parcel AS SELECT row_number() OVER () as gid, properties->'logradouro' AS via_name, \
jgeom AS geom \
FROM pk7600008201201_p2_parcel"
CREATE VIEW
psql postgres://postgres@localhost/ingest20 -c "SELECT ingest.any_load('geojson2sql','/tmp/sandbox/_pkBR821_001/LOTES_URBANOS.geojson','parcel_full','vw2_pk7600008201201_p2_parcel','7600008201201','7fc034ac5c703dcfe93567bd5196ce6eba22df7b779b15ca713b026e85f64759.zip',array[]::text[],5,1)"
NOTICE:  Self-intersection at or near point 775929.49190474558 6530955.2483285191
NOTICE:  Ring Self-intersection at or near point 775630.98803374998 6530612.533643622
NOTICE:  Self-intersection at or near point 776505.44161041756 6531923.4062212706
NOTICE:  Self-intersection at or near point 769891.64836159325 6531766.8143960517
NOTICE:  Self-intersection at or near point 775929.49190474558 6530955.2483285191
NOTICE:  Ring Self-intersection at or near point 775630.98803374998 6530612.533643622
NOTICE:  Self-intersection at or near point 776505.44161041756 6531923.4062212706
NOTICE:  Self-intersection at or near point 769891.64836159325 6531766.8143960517
ERROR:  Geohash requires inputs in decimal degrees, got (779400 6.52982e+06, 779400 6.52982e+06).
CONTEXT:  SQL statement "
      WITH
      scan AS (
        SELECT 23 AS file_id,
                CASE
                 WHEN true THEN gid
                 ELSE gid+COALESCE((SELECT MAX(feature_id) FROM ingest.feature_asis WHERE file_id=23),0)
               END AS gid,
        properties,
               CASE
                 WHEN ST_SRID(geom)=0 THEN ST_SetSRID(geom,4326)
                 WHEN true AND ST_SRID(geom)!=4326 AND true THEN ST_Transform(geom,4326)
                 ELSE geom
               END AS geom
        FROM (
            SELECT gid,  -- feature_id_col
                 to_jsonb(subq) as properties,
                 ST_GeomFromGeoJSON(geom) AS geom -- geom
            FROM vw2_pk7600008201201_p2_parcel , LATERAL (SELECT via_name) subq
          ) t
      ),
      a0 AS (
        SELECT *
        FROM scan
        WHERE ST_IsValid(geom)
      ),
      a AS (
        SELECT *
        FROM a0
        WHERE ST_IsClosed(geom) = TRUE OR GeometryType(geom) IN ('LINESTRING','MULTILINESTRING')
      ),
      mask AS (SELECT ingest.buffer_geom(geom,1) AS geom FROM ingest.vw02full_donated_packfilevers WHERE id=7600008201201 LIMIT 1),
      b AS (
        SELECT file_id, gid, properties, geom, ( B'000000000' ||  (NOT(ST_IsSimple(geom)))::int::bit || (NOT(ST_IsValid(geom)))::int::bit || (NOT(ST_Intersects(geom,(SELECT geom FROM mask))))::int::bit ) AS error_mask
        FROM a
      ),
      c AS (
        (
        SELECT file_id, gid, properties, geom, (error_mask | (
            B'00' ||
            (CASE (SELECT (ingest.donated_PackComponent_geomtype(23))[1]) WHEN 'poly' THEN ST_Area(geom,true) > 2147483647 WHEN 'line' THEN ST_Length(geom,true) > 2147483647 ELSE FALSE END)::int::bit ||
            B'00' ||
            (GeometryType(geom) NOT IN ('POLYGON'   ,'MULTIPOLYGON'))::int::bit ||
            (geom IS NULL)::int::bit ||
            (CASE (SELECT (ingest.donated_PackComponent_geomtype(23))[1]) WHEN 'poly' THEN ST_Area(geom,true) < 5 WHEN 'line' THEN ST_Length(geom,true) < 2 ELSE FALSE END)::int::bit ||
            (ST_IsEmpty(geom))::int::bit ||
            B'000' )) AS error_mask
        FROM (
           SELECT file_id, gid,
                  properties,
                  CASE (SELECT (ingest.donated_PackComponent_geomtype(23))[1])
                    WHEN 'point' THEN ST_ReducePrecision( ST_Intersection(geom,(SELECT geom FROM mask)), 0.000001 )
                    ELSE ST_SimplifyPreserveTopology( -- remove collinear points 
                            ST_ReducePrecision( -- round decimal degrees of SRID 4326, ~1 meter
                              ST_Intersection( geom, (SELECT geom FROM mask) )
                              ,0.000001
                            ),
                            0.00000001
                    )
                  END AS geom,
                  error_mask
           FROM b
           WHERE bit_count(error_mask) = 0 
           ) t
        )
        UNION
        (
            SELECT * FROM b WHERE bit_count(error_mask) <> 0
        )
        UNION
        (
            SELECT file_id, gid, properties, ST_MakeValid(geom) AS geom, B'000100000000' AS error_mask
            FROM a0
            WHERE ST_IsClosed(geom) = FALSE AND GeometryType(geom) NOT IN ('LINESTRING','MULTILINESTRING')
        )
        UNION
        (
            SELECT file_id, gid, properties, ST_MakeValid(geom) AS geom, B'000000000010' AS error_mask
            FROM scan
            WHERE ST_IsValid(geom) = FALSE
        )
      ),
      stats AS (
      SELECT ARRAY [
            (SELECT COUNT(*) FROM scan)::bigint,
            (COUNT(*) filter (WHERE get_bit(error_mask,11) = 1))::bigint, -- intersects
            (COUNT(*) filter (WHERE get_bit(error_mask,10) = 1))::bigint, -- invalid
            (COUNT(*) filter (WHERE get_bit(error_mask, 9) = 1))::bigint, -- simple
            (COUNT(*) filter (WHERE get_bit(error_mask, 8) = 1))::bigint, -- empty
            (COUNT(*) filter (WHERE get_bit(error_mask, 7) = 1))::bigint, -- small
            (COUNT(*) filter (WHERE get_bit(error_mask, 6) = 1))::bigint, -- null
            (COUNT(*) filter (WHERE get_bit(error_mask, 5) = 1))::bigint, -- invalid_type
                                                                          -- bit 4 é reservado para duplicados
            (COUNT(*) filter (WHERE get_bit(error_mask, 3) = 1))::bigint, -- is_closed
            (COUNT(*) filter (WHERE get_bit(error_mask, 2) = 1))::bigint  -- large
                                                                          -- bit 1 uso futuro
                                                                          -- bit 0 uso futuro
            ]
        FROM c
      ),
      ins_asis AS (
        INSERT INTO ingest.feature_asis
        SELECT file_id, gid, properties, geom
        FROM c
            WHERE  bit_count(error_mask) = 0
        RETURNING 1
      ),
      ins_asis_discarded AS (
        INSERT INTO ingest.feature_asis_discarded
        SELECT file_id, gid, properties || jsonb_build_object('error_mask',error_mask), geom
        FROM c
            WHERE  bit_count(error_mask) <> 0 
        RETURNING 1
      )
      SELECT array_append(array_append( (SELECT * FROM stats), (SELECT COUNT(*) FROM ins_asis) ), (SELECT COUNT(*) FROM ins_asis_discarded))
    "
PL/pgSQL function ingest.any_load(text,text,text,text,bigint,text,text[],integer,integer,boolean,text,boolean) line 213 at EXECUTE
make: *** [makefile:158: parcel] Error 1
0e1 commented 2 years ago

Situação interessante.

Ela decorre de um problema na função geojson_readfile_features_jgeom.

Mais precisamente no trecho:

crs || subfeature->'geometry' AS jgeom

Ele não está produzindo um jgeom com o crs. Com isso, ST_GeomFromGeoJSON produz uma geometria com SRID=4326, ao invés de srid=31981:

Enhanced: 3.0.0 parsed geometry defaults to SRID=4326 if not specified otherwise.

Assim, ST_Transform não será aplicada durante o processo de ingestão, transformando de srid=31981 para SRID=4326.

Exemplo da aplicação da função em via de RJ:

SELECT 1, (ROW_NUMBER() OVER())::int AS subfeature_id,
        subfeature->>'type' AS subfeature_type,
        subfeature->'properties' AS properties,
        crs || subfeature->'geometry' AS jgeom
FROM (
SELECT j->>'type' AS geojson_type,
        jsonb_objslice('crs',j) AS crs,
        jsonb_array_elements(j->'features') AS subfeature
FROM ( SELECT pg_read_file('/home/claiton/Logradouros.geojson')::JSONb AS j ) jfile
) t2
LIMIT 1;

 ?column? | subfeature_id | subfeature_type |                                                                                                                                                                                                            properties                                                                                                                                                                                                             |                                                                                    jgeom                                                                                    
----------+---------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
        1 |             1 | Feature         | {"CL": "061069", "Nome": "Centro", "COMPLETO": "Avenida Presidente Vargas", "OBJECTID": 1, "COD_TRECHO": 55403, "Cod_Bairro": 5, "HIERARQUIA": "Arterial primária", "NP_FIN_IMP": null, "NP_FIN_PAR": null, "NP_INI_IMP": null, "NP_INI_PAR": null, "SIT_TRECHO": "Ativo", "CHAVEGEO_TR": "81LFUWE5", "NOME_PARCIAL": "Vargas", "COD_SIT_TRECHO": 1, "COD_TIPO_LOGRA": 5, "TIPO_LOGRA_ABR": "Av   ", "TIPO_LOGRA_EXT": "Avenida"} | {"type": "LineString", "coordinates": [[-43.188256730640092, -22.904302820398094], [-43.187915093230785, -22.904191824180526], [-43.187630425575207, -22.904099266181202]]}
(1 row)

Em vias de RJ não aconteceu o mesmo problema porque os dados estão em SRID=4326.

Exemplo em parcel de Bagé:

SELECT 1, (ROW_NUMBER() OVER())::int AS subfeature_id,
        subfeature->>'type' AS subfeature_type,
        subfeature->'properties' AS properties,
        crs || subfeature->'geometry' AS jgeom
FROM (
SELECT j->>'type' AS geojson_type,
        jsonb_objslice('crs',j) AS crs,
        jsonb_array_elements(j->'features') AS subfeature
FROM ( SELECT pg_read_file('/home/claiton/LOTES_URBANOS.geojson')::JSONb AS j ) jfile
) t2
LIMIT 1;

 ?column? | subfeature_id | subfeature_type |                                                                                                                                                                                                                                                      properties                                                                                                                                                                                                                                                       |                                                                                                                                                    jgeom                                                                                                                                                     
----------+---------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
        1 |             1 | Feature         | {"id": 10856, "baiqd": "0000102504", "compl": "04/9", "bairro": null, "numero": "3120", "codface": null, "terreno": 273.0, "testada": 10.0, "cadastro": "51228", "globalid": "a098ca3c-fc48-49e1-9147-7f923c1fdaaa", "inscricao": "51228", "venal_max": 2926.56, "venal_min": 1575.84, "alerta_alt": "OK", "alerta_bai": "OK", "cadastro_1": "51228", "construida": 21.0, "logradouro": "3 MANUEL CONCEICAO, R", "objet_id_3": "19769", "shape_leng": 83.179144942600004, "valorvenal": 2251.2, "venal_anun": 2251.2} | {"type": "MultiPolygon", "coordinates": [[[[779406.695340814185329, 6529838.014955725520849], [779403.710814313497394, 6529807.374642341397703], [779393.334318355889991, 6529808.486941305920482], [779395.499845423037186, 6529839.118054557591677], [779406.695340814185329, 6529838.014955725520849]]]]}
(1 row)

Exemplo da aplicação de ST_SRID em uma geometria de parcel de Bagé, com e sem crs:

ingest99=# SELECT ST_SRID(ST_GeomFromGeoJSON('{"type": "MultiPolygon", "coordinates": [[[[779406.695340814185329, 6529838.014955725520849], [779403.710814313497394, 6529807.374642341397703], [779393.334318355889991, 6529808.486941305920482], [779395.499845423037186, 6529839.118054557591677], [779406.695340814185329, 6529838.014955725520849]]]], "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::31981"}}}')) AS srid;
 srid  
-------
 31981
(1 row)

ingest99=# SELECT ST_SRID(ST_GeomFromGeoJSON('{"type": "MultiPolygon", "coordinates": [[[[779406.695340814185329, 6529838.014955725520849], [779403.710814313497394, 6529807.374642341397703], [779393.334318355889991, 6529808.486941305920482], [779395.499845423037186, 6529839.118054557591677], [779406.695340814185329, 6529838.014955725520849]]]]}')) AS srid;
 srid 
------
 4326
(1 row)
0e1 commented 2 years ago

@crebollobr

Copie e cole o código a seguir na base que você está usando para Bagé, e teste novamente a ingestão. Ou crie uma ingest nova, dando pull no repositório pg_pubLib-v1 antes). Com isso deve funcionar a ingestão de Bagé, exceto para os casos de o arquivo de origem se enquadrar no caso https://github.com/digital-guard/preserv/issues/104#issuecomment-1115167377.

CREATE or replace FUNCTION geojson_readfile_features_jgeom(file text, file_id int default null) RETURNS TABLE (
  file_id int, feature_id int, feature_type text, properties jsonb, jgeom jsonb
) AS $f$
   SELECT file_id, (ROW_NUMBER() OVER())::int AS subfeature_id,
          subfeature->>'type' AS subfeature_type,
          subfeature->'properties' AS properties,
          crs || (subfeature->'geometry') AS jgeom
   FROM (
      SELECT j->>'type' AS geojson_type,
             jsonb_objslice('crs',j) AS crs,
             jsonb_array_elements(j->'features') AS subfeature
      FROM ( SELECT pg_read_file(file)::JSONb AS j ) jfile
   ) t2
$f$ LANGUAGE SQL;
COMMENT ON FUNCTION geojson_readfile_features_jgeom(text,int)
  IS 'Reads a big GeoJSON file and transforms it into a table with a json-geometry column.'
;
crebollobr commented 2 years ago

Teste OK