sebastic
commented
2 years ago Met deze changes kan het levering_xml record worden toegevoegd:
Stetl
--- /dev/null
+++ b/stetl/filters/bagfilter.py
@@ -0,0 +1,58 @@
+# BAG related filters
+
+from stetl.component import Config
+from stetl.util import Util
+from stetl.filter import Filter
+from stetl.packet import FORMAT
+
+log = Util.get_log("bagfilter")
+
+
+class LeveringFilter(Filter):
+ """
+ Convert Leveringsdocument-BAG-Extract.xml content to record for
+ insertion into nlx_bag_info table.
+ """
+
+ @Config(ptype=str, default='sleutel', required=False)
+ def key_column(self):
+ """
+ Column name for key
+ """
+ pass
+
+ @Config(ptype=str, default='levering_xml', required=False)
+ def key_value(self):
+ """
+ Column value for key
+ """
+ pass
+
+ @Config(ptype=str, default='waarde', required=False)
+ def value_column(self):
+ """
+ Column name for value
+ """
+ pass
+
+ # Constructor
+ def __init__(self, configdict, section, consumes=FORMAT.string, produces=FORMAT.record_array):
+ Filter.__init__(self, configdict, section, consumes, produces)
+
+ def invoke(self, packet):
+ if packet.data is None or packet.is_end_of_stream():
+ return packet
+
+ with open(packet.data, 'rt') as f:
+ data = f.read()
+
+ record = {
+ self.key_column: self.key_value,
+ self.value_column: data,
+ }
+
+ # record_array is used to avoid ValueError:
+ # https://github.com/geopython/stetl/issues/125
+ packet.data = [record]
+
+ return packetNLExtact
--- a/bagv2/etl/conf/etl-imbag-2.1.0.cfg
+++ b/bagv2/etl/conf/etl-imbag-2.1.0.cfg
@@ -28,6 +28,7 @@
# [GDAL_3462]
chains = input_sql_pre|output_postgres_sql,
input_bag_leveringsdoc_file|convert_vsi_to_etree_doc|transformer_xslt_leveringsdoc|output_ogr_postgis,
+ input_bag_leveringsdoc_zipfile|extract_bag_leveringsdoc_file|convert_leveringsdoc_to_record|output_postgres_levering,
input_string_file|command_executor,
input_cbs_gemeenten_csv|output_postgres_gemeenten,
input_bag_wpl_gem_koppeling_file|convert_vsi_to_etree_doc|transformer_xslt_gem_wpl|output_ogr_postgis,
@@ -87,6 +88,11 @@ class = stetl.inputs.fileinput.VsiZipFileInput
file_path = {bag_input_file}
name_filter = Leveringsdocument-BAG-Extract.xml
+[input_bag_leveringsdoc_zipfile]
+class = stetl.inputs.fileinput.ZipFileInput
+file_path = {bag_input_file}
+name_filter = Leveringsdocument-BAG-Extract.xml
+
#
# FILTERS
@@ -98,6 +104,10 @@ class = stetl.filters.fileextractor.VsiFileExtractor
file_path = temp/bagger.{bag_file_ext}
delete_file = True
+[extract_bag_leveringsdoc_file]
+class = stetl.filters.fileextractor.ZipFileExtractor
+file_path = temp/Leveringsdocument-BAG-Extract.xml
+
# Filter to expand a ZIP archive to a configured target dir
[expand_zip_archive]
class = stetl.filters.archiveexpander.ZipArchiveExpander
@@ -111,6 +121,10 @@ class = stetl.filters.formatconverter.FormatConverter
input_format = gdal_vsi_path
output_format = etree_doc
+# convert leverings document content to record
+[convert_leveringsdoc_to_record]
+class = stetl.filters.bagfilter.LeveringFilter
+
# XSLT transformer GEM-WPL-RELATIE to OGR Simple Feature GML doctree
[transformer_xslt_gem_wpl]
class = stetl.filters.xsltfilter.XsltFilter
@@ -148,6 +162,19 @@ table = provincie_gemeente
key = gemeentecode
replace = true
+# Insert levering XML into nlx_bag_info table
+[output_postgres_levering]
+class = stetl.outputs.dboutput.PostgresInsertOutput
+database = {pg_db}
+host = {pg_host}
+port = {pg_port}
+user = {pg_user}
+password = {password}
+schema = {schema}
+table = nlx_bag_info
+key = sleutel
+replace = true
+
# The ogr2ogr command-line for generic GML Features.
[output_ogr_postgis]
class = stetl.outputs.ogroutput.Ogr2OgrOutputGezien de weerstand tegen de changes in #319 zal ik geen pull requests hiervoor indienen. Indien ik niet de enige blijk te zijn die hiervan gebruik wil maken, kan dat als nog.
justb4
Het
Leveringsdocument-BAG-Extract.xmlbestand wordt verwerkt, maar er is geenlevering_xmlrecord voor in denlx_bag_infotabel zoals voor BAG v1:Stetl output van een eerder run: