-
-
Notifications
You must be signed in to change notification settings - Fork 83
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
#272 BRK-DKK v4 verwerking inclusief Pand en ORLabels uit BGT
- Loading branch information
Showing
117 changed files
with
1,437,742 additions
and
339 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # Compiled class file | ||
| *.class | ||
|
|
||
| # Log file | ||
| *.log | ||
|
|
||
| # BlueJ files | ||
| *.ctxt | ||
|
|
||
| # Mobile Tools for Java (J2ME) | ||
| .mtj.tmp/ | ||
|
|
||
| # Package Files # | ||
| *.jar | ||
| *.war | ||
| *.nar | ||
| *.ear | ||
| *.zip | ||
| *.tar.gz | ||
| *.rar | ||
|
|
||
| # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml | ||
| hs_err_pid* | ||
|
|
||
| sync.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,28 +1,33 @@ | ||
| -- Maak schema aan indien deze nog niet bestaat. | ||
| -- Doe niets voor schema 'public' | ||
| CREATE OR REPLACE FUNCTION _nlx_createschema(schemaname VARCHAR) | ||
| RETURNS void AS | ||
| $$ | ||
| BEGIN | ||
| -- CREATE OR REPLACE FUNCTION _nlx_createschema(schemaname VARCHAR) | ||
| -- RETURNS void AS | ||
| -- $$ | ||
| -- BEGIN | ||
| -- | ||
| -- IF schemaname = 'public' THEN | ||
| -- RETURN; | ||
| -- END IF; | ||
| -- | ||
| -- IF EXISTS( | ||
| -- SELECT schema_name | ||
| -- FROM information_schema.schemata | ||
| -- WHERE schema_name = $1 | ||
| -- ) | ||
| -- THEN | ||
| -- EXECUTE 'DROP SCHEMA ' || $1 || ' CASCADE;'; | ||
| -- END IF; | ||
| -- -- EXECUTE 'DROP SCHEMA IF EXISTS ' || $1 || ';'; | ||
| -- EXECUTE 'CREATE SCHEMA ' || $1 || ';'; | ||
| -- | ||
| -- END; | ||
| -- $$ | ||
| -- LANGUAGE plpgsql; | ||
|
|
||
| IF schemaname = 'public' THEN | ||
| RETURN; | ||
| END IF; | ||
| DROP SCHEMA IF EXISTS {schema} CASCADE; | ||
| CREATE SCHEMA {schema}; | ||
|
|
||
| IF NOT EXISTS( | ||
| SELECT schema_name | ||
| FROM information_schema.schemata | ||
| WHERE schema_name = $1 | ||
| ) | ||
| THEN | ||
| EXECUTE 'CREATE SCHEMA ' || $1 || ';'; | ||
| END IF; | ||
|
|
||
| END; | ||
| $$ | ||
| LANGUAGE plpgsql; | ||
|
|
||
| SELECT _nlx_createschema('{schema}'); | ||
|
|
||
| DROP FUNCTION _nlx_createschema(schemaname VARCHAR); | ||
| -- SELECT _nlx_createschema('{schema}'); | ||
| -- | ||
| -- DROP FUNCTION _nlx_createschema(schemaname VARCHAR); | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,77 +1,153 @@ | ||
| -- Auteur: Frank Steggink | ||
| -- Auteur: Frank Steggink - _nlx_dedup_data | ||
| -- Just van den Broecke - feb 2020 - replace by _nlx_dedup_table | ||
| -- | ||
| -- Doel: script om dubbele records te verwijderen | ||
|
|
||
| SET search_path={schema},public; | ||
|
|
||
| DROP TABLE IF EXISTS _nlx_temp; | ||
| -- DROP TABLE IF EXISTS _nlx_temp; | ||
| -- | ||
| -- CREATE OR REPLACE FUNCTION _nlx_dedup_data(tablename VARCHAR) | ||
| -- RETURNS VARCHAR AS | ||
| -- $$ | ||
| -- DECLARE | ||
| -- rowcount INTEGER; | ||
| -- BEGIN | ||
| -- | ||
| -- IF EXISTS (SELECT 1 | ||
| -- FROM information_schema.tables | ||
| -- WHERE table_schema='{schema}' AND table_name=tablename) THEN | ||
| -- | ||
| -- -- IF EXISTS ( | ||
| -- -- SELECT table_name | ||
| -- -- FROM information_schema.tables | ||
| -- -- WHERE table_schema='{schema}' AND table_name=tablename | ||
| -- -- ) | ||
| -- -- THEN | ||
| -- EXECUTE 'WITH ft AS (SELECT * FROM ' || tablename || '), fids AS (SELECT ogc_fid, lokaalid, tijdstipregistratie FROM ft WHERE eindregistratie IS NOT NULL AND lokaalid||tijdstipregistratie IN (SELECT lokaalid||tijdstipregistratie FROM ft WHERE eindregistratie IS NOT NULL GROUP BY lokaalid,tijdstipregistratie HAVING COUNT(*)>1) ORDER BY lokaalid, tijdstipregistratie, lv_publicatiedatum), fids_to_delete AS (SELECT ogc_fid FROM fids WHERE ogc_fid NOT IN (SELECT MIN(ogc_fid) FROM fids GROUP BY lokaalid,tijdstipregistratie)) DELETE FROM ' || tablename || ' WHERE ogc_fid IN (SELECT ogc_fid FROM fids_to_delete);'; | ||
| -- GET DIAGNOSTICS rowcount = ROW_COUNT; | ||
| -- | ||
| -- RETURN tablename || ': ' || rowcount; | ||
| -- ELSE | ||
| -- RETURN tablename || ' does not exist'; | ||
| -- END IF; | ||
| -- | ||
| -- END; | ||
| -- $$ | ||
| -- LANGUAGE plpgsql; | ||
|
|
||
| CREATE OR REPLACE FUNCTION _nlx_dedup_data(tablename VARCHAR) | ||
| -- Voorbeeld: Ontdubbelen voorkomens panden | ||
| -- create table pand_dubbelevoorkomens as select max(ogc_fid) max_fid, lokaalid, tijdstipregistratie, lv_publicatiedatum, count(*) aantal from pand group by lokaalid, tijdstipregistratie, lv_publicatiedatum having count(*) > 1 order by lokaalid, tijdstipregistratie, lv_publicatiedatum; | ||
| -- | ||
| -- create table pand_todelete as select p.ogc_fid from pand p join pand_dubbelevoorkomens d on p.lokaalid=d.lokaalid and p.tijdstipregistratie=d.tijdstipregistratie and p.lv_publicatiedatum=d.lv_publicatiedatum and p.ogc_fid!=d.max_fid; | ||
| -- | ||
| -- delete from pand where ogc_fid in (select ogc_fid from pand_todelete); | ||
| -- drop table pand_dubbelevoorkomens; | ||
| -- drop table pand_todelete; | ||
|
|
||
| -- Crude but effective - JvdB | ||
| CREATE OR REPLACE FUNCTION _nlx_dedup_table(tablename VARCHAR) | ||
| RETURNS VARCHAR AS | ||
| $$ | ||
| DECLARE | ||
| rowcount INTEGER; | ||
| BEGIN | ||
| EXECUTE 'DROP TABLE IF EXISTS _dubbelevoorkomens_' || tablename || ' '; | ||
| EXECUTE 'DROP TABLE IF EXISTS _todelete;'; | ||
|
|
||
| IF EXISTS ( | ||
| SELECT table_name | ||
| FROM information_schema.tables | ||
| WHERE table_schema='{schema}' AND table_name=tablename | ||
| ) | ||
| THEN | ||
| EXECUTE 'WITH ft AS (SELECT * FROM ' || tablename || '), fids AS (SELECT ogc_fid, lokaalid, tijdstipregistratie FROM ft WHERE eindregistratie IS NOT NULL AND lokaalid||tijdstipregistratie IN (SELECT lokaalid||tijdstipregistratie FROM ft WHERE eindregistratie IS NOT NULL GROUP BY lokaalid,tijdstipregistratie HAVING COUNT(*)>1) ORDER BY lokaalid, tijdstipregistratie, lv_publicatiedatum), fids_to_delete AS (SELECT ogc_fid FROM fids WHERE ogc_fid NOT IN (SELECT MIN(ogc_fid) FROM fids GROUP BY lokaalid,tijdstipregistratie)) DELETE FROM ' || tablename || ' WHERE ogc_fid IN (SELECT ogc_fid FROM fids_to_delete);'; | ||
| GET DIAGNOSTICS rowcount = ROW_COUNT; | ||
| EXECUTE 'CREATE TABLE _dubbelevoorkomens_' || tablename || ' AS SELECT max(ogc_fid) max_fid, lokaalid, tijdstipregistratie, lv_publicatiedatum, count(*) aantal FROM ' || tablename || ' GROUP BY lokaalid, tijdstipregistratie, lv_publicatiedatum having count(*) > 1 ORDER BY lokaalid, tijdstipregistratie, lv_publicatiedatum;'; | ||
| EXECUTE 'CREATE TABLE _todelete AS SELECT p.ogc_fid FROM ' || tablename || ' p JOIN _dubbelevoorkomens_' || tablename || ' d ON p.lokaalid=d.lokaalid AND p.tijdstipregistratie=d.tijdstipregistratie AND p.lv_publicatiedatum=d.lv_publicatiedatum AND p.ogc_fid!=d.max_fid;'; | ||
| EXECUTE 'DELETE FROM ' || tablename || ' WHERE ogc_fid IN (SELECT ogc_fid FROM _todelete);'; | ||
| EXECUTE 'DROP TABLE IF EXISTS _dubbelevoorkomens_' || tablename || ' '; | ||
| EXECUTE 'DROP TABLE IF EXISTS _todelete;'; | ||
| RETURN tablename; | ||
| END; | ||
| $$ | ||
| LANGUAGE plpgsql; | ||
|
|
||
| RETURN tablename || ': ' || rowcount; | ||
| ELSE | ||
| RETURN tablename || ' does not exist'; | ||
| END IF; | ||
| -- omdat openbareruimtelabel uit subfeatures is gegenereerd, zijn alle BGT keys/ids sowieso dubbel | ||
| -- _nlx_dedup_table() zou alle dubbele geheel deleten... | ||
| -- dus additionele criteria aanwenden: geometrie_punt en tekst | ||
| CREATE OR REPLACE FUNCTION _nlx_dedup_table_openbareruimtelabel(tablename VARCHAR) | ||
| RETURNS VARCHAR AS | ||
| $$ | ||
| BEGIN | ||
| EXECUTE 'DROP TABLE IF EXISTS _dubbelevoorkomens_' || tablename || ' '; | ||
| EXECUTE 'DROP TABLE IF EXISTS _todelete;'; | ||
|
|
||
| EXECUTE 'CREATE TABLE _dubbelevoorkomens_' || tablename || ' AS SELECT max(ogc_fid) max_fid, lokaalid, tijdstipregistratie, lv_publicatiedatum, geometrie_punt, tekst, count(*) aantal FROM ' || tablename || ' GROUP BY lokaalid, tijdstipregistratie, lv_publicatiedatum, geometrie_punt, tekst having count(*) > 1 ORDER BY lokaalid, tijdstipregistratie, lv_publicatiedatum;'; | ||
| EXECUTE 'CREATE TABLE _todelete AS SELECT p.ogc_fid FROM ' || tablename || ' p JOIN _dubbelevoorkomens_' || tablename || ' d ON p.lokaalid=d.lokaalid AND p.tijdstipregistratie=d.tijdstipregistratie AND p.lv_publicatiedatum=d.lv_publicatiedatum AND p.geometrie_punt=d.geometrie_punt AND p.tekst=d.tekst AND p.ogc_fid!=d.max_fid;'; | ||
| EXECUTE 'DELETE FROM ' || tablename || ' WHERE ogc_fid IN (SELECT ogc_fid FROM _todelete);'; | ||
| EXECUTE 'DROP TABLE IF EXISTS _dubbelevoorkomens_' || tablename || ' '; | ||
| EXECUTE 'DROP TABLE IF EXISTS _todelete;'; | ||
| RETURN tablename; | ||
| END; | ||
| $$ | ||
| LANGUAGE plpgsql; | ||
|
|
||
| SELECT _nlx_dedup_data('bak_tmp'); | ||
| SELECT _nlx_dedup_data('begroeidterreindeel_tmp'); | ||
| SELECT _nlx_dedup_data('bord_tmp'); | ||
| SELECT _nlx_dedup_data('buurt_tmp'); | ||
| SELECT _nlx_dedup_data('functioneelgebied_tmp'); | ||
| SELECT _nlx_dedup_data('gebouwinstallatie_tmp'); | ||
| SELECT _nlx_dedup_data('installatie_tmp'); | ||
| SELECT _nlx_dedup_data('kast_tmp'); | ||
| SELECT _nlx_dedup_data('kunstwerkdeel_tmp'); | ||
| SELECT _nlx_dedup_data('mast_tmp'); | ||
| SELECT _nlx_dedup_data('onbegroeidterreindeel_tmp'); | ||
| SELECT _nlx_dedup_data('ondersteunendwaterdeel_tmp'); | ||
| SELECT _nlx_dedup_data('ondersteunendwegdeel_tmp'); | ||
| SELECT _nlx_dedup_data('ongeclassificeerdobject_tmp'); | ||
| SELECT _nlx_dedup_data('openbareruimte_tmp'); | ||
| SELECT _nlx_dedup_data('openbareruimtelabel'); | ||
| SELECT _nlx_dedup_data('overbruggingsdeel_tmp'); | ||
| SELECT _nlx_dedup_data('overigbouwwerk_tmp'); | ||
| SELECT _nlx_dedup_data('overigescheiding_tmp'); | ||
| SELECT _nlx_dedup_data('paal_tmp'); | ||
| SELECT _nlx_dedup_data('pand_tmp'); | ||
|
|
||
| -- omdat pand_nummeraanduiding uit subfeatures is gegenereerd, zijn alle BGT keys/ids sowieso dubbel | ||
| -- _nlx_dedup_table() zou alle dubbele geheel deleten... | ||
| -- dus additionele criteria aanwenden: geometrie_nummeraanduiding en nummeraanduidingtekst | ||
| CREATE OR REPLACE FUNCTION _nlx_dedup_table_nummeraanduiding(tablename VARCHAR) | ||
| RETURNS VARCHAR AS | ||
| $$ | ||
| BEGIN | ||
| EXECUTE 'DROP TABLE IF EXISTS _dubbelevoorkomens_' || tablename || ' '; | ||
| EXECUTE 'DROP TABLE IF EXISTS _todelete_' || tablename || ' '; | ||
|
|
||
| EXECUTE 'CREATE TABLE _dubbelevoorkomens_' || tablename || ' AS SELECT max(ogc_fid) max_fid, lokaalid, tijdstipregistratie, lv_publicatiedatum, geometrie_nummeraanduiding, nummeraanduidingtekst, count(*) aantal FROM ' || tablename || ' GROUP BY lokaalid, tijdstipregistratie, lv_publicatiedatum, geometrie_nummeraanduiding, nummeraanduidingtekst having count(*) > 1 ORDER BY lokaalid, tijdstipregistratie, lv_publicatiedatum;'; | ||
| EXECUTE 'CREATE TABLE _todelete_' || tablename || ' AS SELECT p.ogc_fid FROM ' || tablename || ' p JOIN _dubbelevoorkomens_' || tablename || ' d ON p.lokaalid=d.lokaalid AND p.tijdstipregistratie=d.tijdstipregistratie AND p.lv_publicatiedatum=d.lv_publicatiedatum AND p.geometrie_nummeraanduiding=d.geometrie_nummeraanduiding AND p.nummeraanduidingtekst=d.nummeraanduidingtekst AND p.ogc_fid!=d.max_fid;'; | ||
| EXECUTE 'DELETE FROM ' || tablename || ' WHERE ogc_fid IN (SELECT ogc_fid FROM _todelete_' || tablename || ');'; | ||
| EXECUTE 'DROP TABLE IF EXISTS _dubbelevoorkomens_' || tablename || ' '; | ||
| EXECUTE 'DROP TABLE IF EXISTS _todelete_' || tablename || ' '; | ||
| RETURN tablename; | ||
| END; | ||
| $$ | ||
| LANGUAGE plpgsql; | ||
|
|
||
| SELECT _nlx_dedup_table('bak'); | ||
| SELECT _nlx_dedup_table('begroeidterreindeel'); | ||
| SELECT _nlx_dedup_table('bord'); | ||
| SELECT _nlx_dedup_table('buurt'); | ||
| SELECT _nlx_dedup_table('functioneelgebied'); | ||
| SELECT _nlx_dedup_table('gebouwinstallatie'); | ||
| SELECT _nlx_dedup_table('installatie'); | ||
| SELECT _nlx_dedup_table('kast'); | ||
| SELECT _nlx_dedup_table('kunstwerkdeel'); | ||
| SELECT _nlx_dedup_table('mast'); | ||
| SELECT _nlx_dedup_table('onbegroeidterreindeel'); | ||
| SELECT _nlx_dedup_table('ondersteunendwaterdeel'); | ||
| SELECT _nlx_dedup_table('ondersteunendwegdeel'); | ||
| SELECT _nlx_dedup_table('ongeclassificeerdobject'); | ||
| SELECT _nlx_dedup_table('openbareruimte'); | ||
| SELECT _nlx_dedup_table_openbareruimtelabel('openbareruimtelabel'); --subfeatures komen per definitie meerdere keren voor | ||
| SELECT _nlx_dedup_table('overbruggingsdeel'); | ||
| SELECT _nlx_dedup_table('overigbouwwerk'); | ||
| SELECT _nlx_dedup_table('overigescheiding'); | ||
| SELECT _nlx_dedup_table('paal'); | ||
| SELECT _nlx_dedup_table('pand'); | ||
| SELECT _nlx_dedup_table_nummeraanduiding('pand_nummeraanduiding'); --subfeatures komen per definitie meerdere keren voor | ||
|
|
||
| -- Plaatsbepalingspunten tijdelijk uitgeschakeld, bevatten geen tijdstipregistratie en zitten niet | ||
| -- in de dump. | ||
| --SELECT _nlx_dedup_data('plaatsbepalingspunt'); | ||
| --SELECT _nlx_dedup_table('plaatsbepalingspunt'); | ||
|
|
||
| SELECT _nlx_dedup_data('put_tmp'); | ||
| SELECT _nlx_dedup_data('scheiding_tmp'); | ||
| SELECT _nlx_dedup_data('sensor_tmp'); | ||
| SELECT _nlx_dedup_data('spoor_tmp'); | ||
| SELECT _nlx_dedup_data('stadsdeel_tmp'); | ||
| SELECT _nlx_dedup_data('straatmeubilair_tmp'); | ||
| SELECT _nlx_dedup_data('tunneldeel_tmp'); | ||
| SELECT _nlx_dedup_data('vegetatieobject_tmp'); | ||
| SELECT _nlx_dedup_data('waterdeel_tmp'); | ||
| SELECT _nlx_dedup_data('waterinrichtingselement_tmp'); | ||
| SELECT _nlx_dedup_data('waterschap_tmp'); | ||
| SELECT _nlx_dedup_data('wegdeel_tmp'); | ||
| SELECT _nlx_dedup_data('weginrichtingselement_tmp'); | ||
| SELECT _nlx_dedup_data('wijk_tmp'); | ||
| SELECT _nlx_dedup_table('put'); | ||
| SELECT _nlx_dedup_table('scheiding'); | ||
| SELECT _nlx_dedup_table('sensor'); | ||
| SELECT _nlx_dedup_table('spoor'); | ||
| SELECT _nlx_dedup_table('stadsdeel'); | ||
| SELECT _nlx_dedup_table('straatmeubilair'); | ||
| SELECT _nlx_dedup_table('tunneldeel'); | ||
| SELECT _nlx_dedup_table('vegetatieobject'); | ||
| SELECT _nlx_dedup_table('waterdeel'); | ||
| SELECT _nlx_dedup_table('waterinrichtingselement'); | ||
| SELECT _nlx_dedup_table('waterschap'); | ||
| SELECT _nlx_dedup_table('wegdeel'); | ||
| SELECT _nlx_dedup_table('weginrichtingselement'); | ||
| SELECT _nlx_dedup_table('wijk'); | ||
|
|
||
| DROP FUNCTION _nlx_dedup_data(tablename VARCHAR); | ||
| DROP FUNCTION _nlx_dedup_table(tablename VARCHAR); | ||
| DROP FUNCTION _nlx_dedup_table_openbareruimtelabel(tablename VARCHAR); | ||
| DROP FUNCTION _nlx_dedup_table_nummeraanduiding(tablename VARCHAR); | ||
|
|
||
| SET search_path="$user",public; | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.