Skip to content

Commit d606693

Browse files
leijurvleijurv
committed
pure lua version
1 parent 73ec715 commit d606693

7 files changed

Lines changed: 164 additions & 817 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,6 @@ else()
264264
src/gen/gen-base.cpp
265265
src/gen/gen-create.cpp
266266
src/gen/gen-discrete-isolation.cpp
267-
src/gen/gen-grouped-linemerge.cpp
268267
src/gen/gen-rivers.cpp
269268
src/gen/gen-tile-builtup.cpp
270269
src/gen/gen-tile-raster.cpp

‎flex-config/gen/grouped-linemerge.lua‎

Lines changed: 162 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,106 +1,202 @@
11
-- This config example file is released into the Public Domain.
22
--
3-
-- This Lua config demonstrates the 'grouped-linemerge' generalization. It
4-
-- merges connected lines that share the same set of grouping columns into
5-
-- single (multi-)lines, the equivalent of
3+
-- 'grouped-linemerge' implemented as a PURE FLEX STYLE -- no C++ generalizer
4+
-- strategy, just osm2pgsql.run_sql() in process_gen(). It merges connected
5+
-- lines that share the same grouping columns into single (multi)lines, the
6+
-- equivalent of
67
--
78
-- SELECT cols..., (ST_Dump(ST_LineMerge(ST_Collect(geom)))).geom
89
-- FROM roads GROUP BY cols...
910
--
10-
-- but done globally and maintained incrementally on updates. A typical use is
11-
-- merging road segments that render identically (same name/ref/highway/layer)
12-
-- so that labels and route shields are placed on the whole road instead of on
13-
-- each individual OSM way, without the artifacts you get when merging only
14-
-- within a tile.
11+
-- built once and maintained incrementally on updates, using ordinary tile
12+
-- expiry as the "what changed" signal. A typical use is merging road segments
13+
-- that render identically (same name/ref/highway/layer) so labels and shields
14+
-- sit on the whole road instead of on each OSM way.
1515
--
16-
-- NOTE THAT THE GENERALIZATION SUPPORT IS EXPERIMENTAL AND MIGHT CHANGE
17-
-- WITHOUT NOTICE!
18-
--
19-
-- Workflow:
20-
-- * Import as usual: osm2pgsql -O flex -S grouped-linemerge.lua DATA.osm.pbf
21-
-- * Build the merged table: osm2pgsql-gen -S grouped-linemerge.lua
22-
-- * Apply an update: osm2pgsql -a -O flex -S grouped-linemerge.lua CHANGES.osc.gz
23-
-- * Update the merged table: osm2pgsql-gen -a -S grouped-linemerge.lua
16+
-- Workflow (same as any gen config -- run osm2pgsql-gen after osm2pgsql):
17+
-- Import: osm2pgsql -O flex -S grouped-linemerge.lua DATA.osm.pbf
18+
-- Build merged table: osm2pgsql-gen -S grouped-linemerge.lua
19+
-- Apply an update: osm2pgsql -a -O flex -S grouped-linemerge.lua CHANGES.osc.gz
20+
-- Update merged table: osm2pgsql-gen -a -S grouped-linemerge.lua
2421

25-
-- An expire output records which tiles changed during an update. The
26-
-- grouped-linemerge generalization uses it only as a seed for "where did line
27-
-- geometry change" - it then walks each affected connected component out from
28-
-- there and re-merges it. Use a high maxzoom so the seed regions are small.
29-
local exp_roads = osm2pgsql.define_expire_output({
30-
maxzoom = 18,
31-
table = 'exp_roads',
32-
})
22+
-- ---------------------------------------------------------------------------
23+
-- Configuration
24+
-- ---------------------------------------------------------------------------
25+
local SRC = 'roads' -- source table (one row per OSM way)
26+
local DEST = 'roads_merged' -- destination table (the merged lines)
27+
local EXPIRE = 'exp_roads' -- tile-expire table (the change signal)
28+
local GEOM = 'geom' -- geometry column (same name in src and dest)
29+
local ZOOM = 18 -- expire zoom; high so the changed regions are small
30+
-- Lines merge only when ALL of these columns are equal (NULLs compare equal):
31+
local GROUP = { 'name', 'ref', 'highway', 'layer' }
32+
-- Optional pre-filter: lines not matching are excluded entirely.
33+
local WHERE = 'name IS NOT NULL OR ref IS NOT NULL'
34+
35+
-- ---------------------------------------------------------------------------
36+
-- Build the SQL fragments from the grouping columns (this is the only thing
37+
-- the C++ strategy did that wasn't already plain SQL -- and it's trivial here).
38+
-- ---------------------------------------------------------------------------
39+
local function join(sep, fn)
40+
local t = {}
41+
for _, c in ipairs(GROUP) do t[#t + 1] = fn(c) end
42+
return table.concat(t, sep)
43+
end
44+
45+
local V = {
46+
src = SRC, dest = DEST, expire = EXPIRE, geom = GEOM, zoom = ZOOM,
47+
where = '(' .. WHERE .. ')',
48+
group_cols = join(', ', function(c) return '"' .. c .. '"' end),
49+
group_cols_l = join(', ', function(c) return 'l."' .. c .. '"' end),
50+
group_cols_gk = join(', ', function(c) return '"' .. c .. '" AS "gk_' .. c .. '"' end),
51+
group_cols_l_gk = join(', ', function(c) return 'l."' .. c .. '" AS "gk_' .. c .. '"' end),
52+
group_join = join(' AND ', function(c) return 'l."' .. c .. '" IS NOT DISTINCT FROM n."gk_' .. c .. '"' end),
53+
group_join_dn = join(' AND ', function(c) return 'd."' .. c .. '" IS NOT DISTINCT FROM n."gk_' .. c .. '"' end),
54+
}
55+
56+
local function sql(s)
57+
return (s:gsub('{([%w_]+)}', function(k)
58+
local v = V[k]
59+
if v == nil then error("unknown template key '" .. k .. "'") end
60+
return tostring(v)
61+
end))
62+
end
63+
64+
-- ---------------------------------------------------------------------------
65+
-- Tables (used by osm2pgsql on import; registered by osm2pgsql-gen)
66+
-- ---------------------------------------------------------------------------
67+
local exp_roads = osm2pgsql.define_expire_output({ table = EXPIRE, maxzoom = ZOOM })
3368

34-
-- The source table with the original road segments (one row per OSM way).
3569
local roads = osm2pgsql.define_table({
36-
name = 'roads',
70+
name = SRC,
3771
ids = { type = 'way', id_column = 'way_id' },
3872
columns = {
3973
{ column = 'name', type = 'text' },
4074
{ column = 'ref', type = 'text' },
4175
{ column = 'highway', type = 'text' },
4276
{ column = 'layer', type = 'int' },
43-
-- Attach the expire output to the geometry so that any change to a
44-
-- road's geometry (add/modify/delete) expires the tiles it covers.
45-
{ column = 'geom', type = 'linestring', not_null = true,
77+
-- Any geometry change (add/modify/delete) expires the tiles it covers.
78+
{ column = GEOM, type = 'linestring', not_null = true,
4679
expire = { { output = exp_roads } } },
4780
}
4881
})
4982

50-
-- The destination table with the merged roads. Its columns are exactly the
51-
-- grouping columns plus the geometry. It has no OSM id column (it is derived
52-
-- data maintained by osm2pgsql-gen, not by the normal update process); the
53-
-- warning osm2pgsql prints about that is expected.
5483
osm2pgsql.define_table({
55-
name = 'roads_merged',
84+
name = DEST,
5685
columns = {
5786
{ column = 'name', type = 'text' },
5887
{ column = 'ref', type = 'text' },
5988
{ column = 'highway', type = 'text' },
6089
{ column = 'layer', type = 'int' },
61-
{ column = 'geom', type = 'linestring', not_null = true },
90+
{ column = GEOM, type = 'linestring', not_null = true },
6291
}
6392
})
6493

6594
function osm2pgsql.process_way(object)
66-
local highway = object.tags.highway
67-
if not highway then
68-
return
69-
end
95+
if not object.tags.highway then return end
7096
roads:insert({
7197
name = object.tags.name,
7298
ref = object.tags.ref,
73-
highway = highway,
99+
highway = object.tags.highway,
74100
layer = tonumber(object.tags.layer),
75101
geom = object:as_linestring(),
76102
})
77103
end
78104

105+
-- ---------------------------------------------------------------------------
106+
-- The generalization, in SQL. process_gen() runs in osm2pgsql-gen; it branches
107+
-- on osm2pgsql.mode ('create' for the full build, 'append' for updates).
108+
-- ---------------------------------------------------------------------------
79109
function osm2pgsql.process_gen()
80-
osm2pgsql.run_gen('grouped-linemerge', {
81-
name = 'roads', -- name (for logging)
82-
debug = false, -- set to true for more detailed debug output
83-
src_table = 'roads', -- input table with the line segments
84-
dest_table = 'roads_merged', -- output table for the merged lines
85-
geom_column = 'geom', -- geometry column (same in src and dest)
86-
87-
-- Lines are merged when ALL of these columns are equal (NULLs compare
88-
-- equal). Pass them as a comma-separated list.
89-
group_by_columns = 'name, ref, highway, layer',
90-
91-
-- Optional pre-filter (SQL boolean expression on the source columns).
92-
-- Lines not matching are completely excluded from the generalization.
93-
-- Here we only merge roads that carry a label or a shield.
94-
where = 'name IS NOT NULL OR ref IS NOT NULL',
95-
96-
-- In append mode, where to read the expired tiles from, and the zoom
97-
-- level they were captured at (must match the expire output's maxzoom).
98-
expire_list = 'exp_roads',
99-
zoom = 18,
100-
101-
-- Create functional endpoint indexes on the src/dest tables in create
102-
-- mode. These make the incremental component walk fast. Set to false
103-
-- if you manage the indexes yourself.
104-
create_indexes = true,
105-
})
110+
if osm2pgsql.mode == 'create' then
111+
-- One global GROUP BY + ST_LineMerge over the whole source table.
112+
osm2pgsql.run_sql({
113+
description = 'grouped-linemerge: full rebuild',
114+
transaction = true,
115+
sql = {
116+
sql('TRUNCATE {dest}'),
117+
sql([[INSERT INTO {dest} ({group_cols}, "{geom}")
118+
SELECT {group_cols}, (ST_Dump(ST_LineMerge(ST_Collect("{geom}")))).geom
119+
FROM {src} WHERE {where} GROUP BY {group_cols}]]),
120+
}
121+
})
122+
-- Functional endpoint indexes that make the incremental walk fast.
123+
osm2pgsql.run_sql({
124+
description = 'grouped-linemerge: endpoint indexes',
125+
sql = {
126+
sql([[CREATE INDEX IF NOT EXISTS "{src}_glm_startpt"
127+
ON {src} USING btree (ST_StartPoint("{geom}")) WHERE {where}]]),
128+
sql([[CREATE INDEX IF NOT EXISTS "{src}_glm_endpt"
129+
ON {src} USING btree (ST_EndPoint("{geom}")) WHERE {where}]]),
130+
sql('ANALYZE {dest}'),
131+
sql('CREATE INDEX IF NOT EXISTS "{dest}_glm_startpt" ON {dest} USING btree (ST_StartPoint("{geom}"))'),
132+
sql('CREATE INDEX IF NOT EXISTS "{dest}_glm_endpt" ON {dest} USING btree (ST_EndPoint("{geom}"))'),
133+
}
134+
})
135+
else
136+
-- Incremental: consume the expired tiles, walk each affected connected
137+
-- component out from there, and re-merge only those. Everything is one
138+
-- run_sql with transaction=true so the ON COMMIT DROP temp tables
139+
-- survive across the steps; if_has_rows makes it a no-op when nothing
140+
-- expired.
141+
osm2pgsql.run_sql({
142+
description = 'grouped-linemerge: incremental update',
143+
transaction = true,
144+
if_has_rows = sql('SELECT 1 FROM {expire} WHERE zoom = {zoom} LIMIT 1'),
145+
sql = {
146+
-- 1. expired tiles -> changed-region envelopes
147+
sql([[CREATE TEMP TABLE _glm_region ON COMMIT DROP AS
148+
WITH expired AS (DELETE FROM {expire} WHERE zoom = {zoom} RETURNING x, y)
149+
SELECT ST_TileEnvelope({zoom}, x, y) AS env FROM expired]]),
150+
'ANALYZE _glm_region',
151+
-- 2. walk: seed from lines in the region, flood shared endpoints
152+
-- within the same grouping key (dedup on (group, point)).
153+
sql([[CREATE TEMP TABLE _glm_nodes ON COMMIT DROP AS
154+
WITH RECURSIVE
155+
seeds AS (
156+
SELECT {group_cols_l}, l."{geom}"
157+
FROM _glm_region r
158+
JOIN {src} l ON l."{geom}" && r.env AND ST_Intersects(l."{geom}", r.env)
159+
WHERE {where}
160+
),
161+
nodes AS (
162+
SELECT b.* FROM (
163+
SELECT {group_cols_gk}, ST_StartPoint("{geom}") AS pt FROM seeds
164+
UNION
165+
SELECT {group_cols_gk}, ST_EndPoint("{geom}") FROM seeds
166+
) b
167+
UNION
168+
SELECT {group_cols_l_gk},
169+
CASE WHEN ST_StartPoint(l."{geom}") = n.pt
170+
THEN ST_EndPoint(l."{geom}") ELSE ST_StartPoint(l."{geom}") END
171+
FROM nodes n
172+
JOIN {src} l
173+
ON {group_join}
174+
AND ( ST_StartPoint(l."{geom}") = n.pt OR ST_EndPoint(l."{geom}") = n.pt )
175+
AND {where}
176+
)
177+
SELECT * FROM nodes]]),
178+
'ANALYZE _glm_nodes',
179+
-- 3. collect the member lines of those components
180+
sql([[CREATE TEMP TABLE _glm_ways ON COMMIT DROP AS
181+
SELECT DISTINCT ON (l.ctid) {group_cols_l}, l."{geom}"
182+
FROM _glm_nodes n
183+
JOIN {src} l
184+
ON {group_join}
185+
AND ( ST_StartPoint(l."{geom}") = n.pt OR ST_EndPoint(l."{geom}") = n.pt )
186+
AND {where}
187+
ORDER BY l.ctid]]),
188+
-- 4a. delete stale outputs by reached node (exact endpoint)
189+
sql([[DELETE FROM {dest} d USING _glm_nodes n
190+
WHERE {group_join_dn}
191+
AND ( ST_StartPoint(d."{geom}") = n.pt OR ST_EndPoint(d."{geom}") = n.pt )]]),
192+
-- 4b. clean up components that vanished entirely (region pass)
193+
sql([[DELETE FROM {dest} d USING _glm_region r
194+
WHERE d."{geom}" && r.env AND ST_Intersects(d."{geom}", r.env)]]),
195+
-- 5. regenerate the affected components
196+
sql([[INSERT INTO {dest} ({group_cols}, "{geom}")
197+
SELECT {group_cols}, (ST_Dump(ST_LineMerge(ST_Collect("{geom}")))).geom
198+
FROM _glm_ways GROUP BY {group_cols}]]),
199+
}
200+
})
201+
end
106202
end

‎src/gen/gen-create.cpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
#include "format.hpp"
1313
#include "gen-base.hpp"
1414
#include "gen-discrete-isolation.hpp"
15-
#include "gen-grouped-linemerge.hpp"
1615
#include "gen-rivers.hpp"
1716
#include "gen-tile-builtup.hpp"
1817
#include "gen-tile-raster.hpp"
@@ -32,10 +31,6 @@ std::unique_ptr<gen_base_t> create_generalizer(std::string const &strategy,
3231
if (strategy == "discrete-isolation") {
3332
return std::make_unique<gen_di_t>(connection, append, params);
3433
}
35-
if (strategy == "grouped-linemerge") {
36-
return std::make_unique<gen_grouped_linemerge_t>(connection, append,
37-
params);
38-
}
3934
if (strategy == "raster-union") {
4035
return std::make_unique<gen_tile_raster_union_t>(connection, append,
4136
params);

0 commit comments

Comments
 (0)