first commit
This commit is contained in:
Fabio Micheluz
commit
acf5948fdb
7 changed files with 542 additions and 0 deletions
7
f1.sh
Executable file
7
f1.sh
Executable file
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
date
|
||||
echo "79350"
|
||||
#wc -l manifest_dem.txt
|
||||
#find terrain_rgb -type f \( -name "*.tif" -o -name "*.tiff" \) | wc -l
|
||||
cat resume/dem_resume.txt
|
||||
du -sh terrain_rgb
|
||||
8
f2.sh
Executable file
8
f2.sh
Executable file
|
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
date
|
||||
echo "26450"
|
||||
#wc -l manifest_dem.txt
|
||||
#find terrain_rgb -type f \( -name "*.tif" -o -name "*.tiff" \) | wc -l
|
||||
find resume/tiles -type f -name "*.done" | wc -l
|
||||
#cat resume/dem_resume.txt
|
||||
du -sh tiles
|
||||
93
make_tiles_italy.sh
Executable file
93
make_tiles_italy.sh
Executable file
|
|
@ -0,0 +1,93 @@
|
|||
#!/bin/zsh
|
||||
|
||||
###############################################
|
||||
# CONFIG
|
||||
###############################################
|
||||
|
||||
PYTHON="/usr/bin/python3"
|
||||
|
||||
TERRAIN_ROOT="/Volumes/terrain/tiles"
|
||||
TILES_ROOT="/Volumes/SSD4T/tiles_italy_new"
|
||||
LOG_DIR="/Volumes/SSD4T/logs_new"
|
||||
RESUME_DIR="/Volumes/SSD4T/resume_new"
|
||||
|
||||
Z_MIN=7
|
||||
Z_MAX=14
|
||||
|
||||
mkdir -p "$TILES_ROOT" "$LOG_DIR" "$RESUME_DIR"
|
||||
|
||||
|
||||
###############################################
|
||||
# CHECK PYTHON
|
||||
###############################################
|
||||
|
||||
if [ ! -x "$PYTHON" ]; then
|
||||
echo "[ERRORE] Python non trovato: $PYTHON"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
###############################################
|
||||
# LOOP SU TUTTI I DEM ITALIANI
|
||||
###############################################
|
||||
|
||||
DEM_LIST=$(find "$TERRAIN_ROOT" -maxdepth 1 -type d -name "Copernicus_DSM_COG_10_*_DEM" \
|
||||
| grep -E "N(3[6-9]|4[0-7])_00_E(00[6-9]|0(1[0-9]))_00" \
|
||||
| sort)
|
||||
|
||||
echo "[INFO] Trovati $(echo "$DEM_LIST" | wc -l | tr -d ' ') DEM italiani"
|
||||
|
||||
for DEM_DIR in $DEM_LIST; do
|
||||
BASENAME=$(basename "$DEM_DIR")
|
||||
TIF="$DEM_DIR/${BASENAME}_terrain_rgb.tif"
|
||||
|
||||
if [ ! -f "$TIF" ]; then
|
||||
echo "[WARN] Nessun TIFF TerrainRGB in $DEM_DIR"
|
||||
continue
|
||||
fi
|
||||
|
||||
OUT_DIR="$TILES_ROOT/$BASENAME"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
RESUME_FILE="$RESUME_DIR/${BASENAME}.done"
|
||||
LOGFILE="$LOG_DIR/${BASENAME}.log"
|
||||
|
||||
if [ -f "$RESUME_FILE" ]; then
|
||||
echo "[SKIP] Tiles già generati per $BASENAME"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[INFO] Genero tiles per $BASENAME"
|
||||
|
||||
{
|
||||
echo "----------------------------------------"
|
||||
echo "DEM: $BASENAME"
|
||||
echo "Input: $TIF"
|
||||
echo "Output: $OUT_DIR"
|
||||
echo "Zoom: $Z_MIN-$Z_MAX"
|
||||
echo "Start: $(date)"
|
||||
echo "----------------------------------------"
|
||||
|
||||
CORES=$(sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
||||
|
||||
$PYTHON terrain_rgb_to_tiles.py "$TIF" "$OUT_DIR" "$Z_MIN" "$Z_MAX" "$CORES"
|
||||
STATUS=$?
|
||||
|
||||
echo "End: $(date)"
|
||||
echo "Status: $STATUS"
|
||||
echo "----------------------------------------"
|
||||
|
||||
} >> "$LOGFILE" 2>&1
|
||||
|
||||
if [ "$STATUS" -ne 0 ]; then
|
||||
echo "[ERRORE] Tiles falliti per $BASENAME"
|
||||
continue
|
||||
fi
|
||||
|
||||
touch "$RESUME_FILE"
|
||||
echo "[OK] Tiles creati per $BASENAME"
|
||||
|
||||
done
|
||||
|
||||
echo "[INFO] Completato!"
|
||||
|
||||
35
merge_tiles.sh
Executable file
35
merge_tiles.sh
Executable file
|
|
@ -0,0 +1,35 @@
|
|||
#!/bin/zsh
|
||||
|
||||
SRC_ROOT="tiles_italy_new"
|
||||
DST_ROOT="merged_tiles"
|
||||
|
||||
mkdir -p "$DST_ROOT"
|
||||
|
||||
echo "[INFO] Merge tiles da $SRC_ROOT → $DST_ROOT"
|
||||
|
||||
# Scorri tutte le tile z/x/y.png in tutte le sottocartelle
|
||||
find "$SRC_ROOT" -type f -name '*.png' | while read -r TILE; do
|
||||
# TILE = tiles_italy_new/<DEM>/<z>/<x>/<y>.png
|
||||
REL="${TILE#$SRC_ROOT/}" # <DEM>/<z>/<x>/<y>.png
|
||||
DEM="${REL%%/*}" # <DEM>
|
||||
REST="${REL#*/}" # <z>/<x>/<y>.png
|
||||
|
||||
Z="$(echo "$REST" | cut -d'/' -f1)"
|
||||
X="$(echo "$REST" | cut -d'/' -f2)"
|
||||
Y_FILE="$(echo "$REST" | cut -d'/' -f3)"
|
||||
|
||||
OUT_DIR="$DST_ROOT/$Z/$X"
|
||||
OUT_PATH="$OUT_DIR/$Y_FILE"
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
if [ -f "$OUT_PATH" ]; then
|
||||
echo "[WARN] Tile già esistente, lascio la prima:
|
||||
$OUT_PATH"
|
||||
continue
|
||||
fi
|
||||
|
||||
cp "$TILE" "$OUT_PATH"
|
||||
done
|
||||
|
||||
echo "[INFO] Merge completato in $DST_ROOT"
|
||||
113
ph1.sh
Executable file
113
ph1.sh
Executable file
|
|
@ -0,0 +1,113 @@
|
|||
#!/bin/zsh
|
||||
|
||||
###############################################
|
||||
# CONFIG
|
||||
###############################################
|
||||
|
||||
RIO="/opt/local/Library/Frameworks/Python.framework/Versions/3.11/bin/rio"
|
||||
MANIFEST="manifest_dem.txt"
|
||||
|
||||
OUT_ROOT="terrain_rgb"
|
||||
LOG_DIR="logs/dem"
|
||||
RESUME_FILE="resume/dem_resume.txt"
|
||||
|
||||
mkdir -p "$OUT_ROOT" "$LOG_DIR" "resume"
|
||||
|
||||
|
||||
###############################################
|
||||
# CHECK BINARI
|
||||
###############################################
|
||||
|
||||
check_bin() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "[ERRORE] Binario non trovato: $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_bin gdalinfo
|
||||
check_bin gdal_translate
|
||||
|
||||
if [ ! -x "$RIO" ]; then
|
||||
echo "[ERRORE] rio non trovato: $RIO"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
###############################################
|
||||
# RESUME
|
||||
###############################################
|
||||
|
||||
LAST_DONE=0
|
||||
if [ -f "$RESUME_FILE" ]; then
|
||||
LAST_DONE=$(cat "$RESUME_FILE")
|
||||
fi
|
||||
|
||||
DEM_COUNT=0
|
||||
|
||||
|
||||
###############################################
|
||||
# LOOP DEM
|
||||
###############################################
|
||||
|
||||
while IFS= read -r DEM; do
|
||||
|
||||
DEM=$(echo "$DEM" | tr -d '\r' | xargs)
|
||||
[ -z "$DEM" ] && continue
|
||||
|
||||
if [ ! -f "$DEM" ]; then
|
||||
echo "[WARN] DEM non trovato: $DEM"
|
||||
continue
|
||||
fi
|
||||
|
||||
DEM_COUNT=$((DEM_COUNT + 1))
|
||||
|
||||
if [ "$DEM_COUNT" -le "$LAST_DONE" ]; then
|
||||
echo "[SKIP] DEM #$DEM_COUNT già processato"
|
||||
continue
|
||||
fi
|
||||
|
||||
BASENAME=$(basename "$DEM" .tif)
|
||||
WORKDIR="$OUT_ROOT/$BASENAME"
|
||||
mkdir -p "$WORKDIR"
|
||||
|
||||
LOGFILE="$LOG_DIR/${BASENAME}.log"
|
||||
|
||||
echo "[INFO] DEM #$DEM_COUNT → $DEM"
|
||||
echo "[INFO] Log: $LOGFILE"
|
||||
|
||||
OUT_TERRAIN="$WORKDIR/${BASENAME}_terrain_rgb.tif"
|
||||
|
||||
{
|
||||
echo "----------------------------------------"
|
||||
echo "DEM: $DEM"
|
||||
echo "Output: $OUT_TERRAIN"
|
||||
echo "Start: $(date)"
|
||||
echo "----------------------------------------"
|
||||
|
||||
"$RIO" rgbify \
|
||||
--min-z 0 \
|
||||
--max-z 9000 \
|
||||
--format png \
|
||||
"$DEM" \
|
||||
"$OUT_TERRAIN"
|
||||
|
||||
STATUS=$?
|
||||
|
||||
echo "End: $(date)"
|
||||
echo "Status: $STATUS"
|
||||
echo "----------------------------------------"
|
||||
|
||||
} >> "$LOGFILE" 2>&1
|
||||
|
||||
if [ "$STATUS" -ne 0 ]; then
|
||||
echo "[ERRORE] rgbify fallito per $DEM"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "$DEM_COUNT" > "$RESUME_FILE"
|
||||
echo "[OK] Terrain‑RGB generato per $BASENAME"
|
||||
|
||||
done < "$MANIFEST"
|
||||
|
||||
echo "[INFO] Fase 1 completata"
|
||||
238
ph2.sh
Executable file
238
ph2.sh
Executable file
|
|
@ -0,0 +1,238 @@
|
|||
|
||||
#!/bin/zsh
|
||||
|
||||
###############################################
|
||||
# CONFIG
|
||||
###############################################
|
||||
|
||||
PYTHON="/opt/local/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
|
||||
|
||||
TERRAIN_ROOT="terrain_rgb"
|
||||
TILES_ROOT="tiles"
|
||||
LOG_DIR="logs/tiles"
|
||||
RESUME_DIR="resume/tiles"
|
||||
|
||||
Z_MIN=5
|
||||
Z_MAX=14
|
||||
|
||||
mkdir -p "$TILES_ROOT" "$LOG_DIR" "$RESUME_DIR"
|
||||
|
||||
|
||||
###############################################
|
||||
# CHECK PYTHON
|
||||
###############################################
|
||||
|
||||
if [ ! -x "$PYTHON" ]; then
|
||||
echo "[ERRORE] Python non trovato: $PYTHON"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
###############################################
|
||||
# PYTHON SCRIPT INLINE (NUOVO E CORRETTO)
|
||||
###############################################
|
||||
|
||||
PY_SCRIPT="$(mktemp)"
|
||||
cat > "$PY_SCRIPT" << 'EOF'
|
||||
import os, sys, math
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
import rasterio
|
||||
from rasterio.warp import transform_bounds
|
||||
from rasterio.enums import Resampling
|
||||
from rasterio.windows import Window
|
||||
from PIL import Image
|
||||
|
||||
|
||||
# ---------- Tile math ----------
|
||||
def latlon_to_tile(lat, lon, z):
|
||||
lat_rad = math.radians(lat)
|
||||
n = 2 ** z
|
||||
xtile = int((lon + 180) / 360 * n)
|
||||
ytile = int((1 - math.log(math.tan(lat_rad) + 1 / math.cos(lat_rad)) / math.pi) / 2 * n)
|
||||
return xtile, ytile
|
||||
|
||||
def tile_bounds_wgs84(z, x, y):
|
||||
n = 2 ** z
|
||||
lon_min = x / n * 360 - 180
|
||||
lon_max = (x + 1) / n * 360 - 180
|
||||
lat_min = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * (y + 1) / n))))
|
||||
lat_max = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * y / n))))
|
||||
return lon_min, lat_min, lon_max, lat_max
|
||||
|
||||
|
||||
# ---------- Clamp window ----------
|
||||
def clamp_window(window, src):
|
||||
row_off = max(0, int(window.row_off))
|
||||
col_off = max(0, int(window.col_off))
|
||||
height = max(0, int(min(window.height, src.height - row_off)))
|
||||
width = max(0, int(min(window.width, src.width - col_off)))
|
||||
if width <= 0 or height <= 0:
|
||||
return None
|
||||
return Window(col_off, row_off, width, height)
|
||||
|
||||
|
||||
# ---------- Tile job ----------
|
||||
def process_tile(args):
|
||||
tif_path, tiles_root, z, x, y = args
|
||||
try:
|
||||
with rasterio.open(tif_path) as src:
|
||||
# CRS check
|
||||
if src.crs is None:
|
||||
return False
|
||||
|
||||
# Tile bounds in WGS84
|
||||
lon_min, lat_min, lon_max, lat_max = tile_bounds_wgs84(z, x, y)
|
||||
|
||||
# Transform tile bounds raster CRS
|
||||
left, bottom, right, top = transform_bounds(
|
||||
"EPSG:4326", src.crs, lon_min, lat_min, lon_max, lat_max, densify_pts=0
|
||||
)
|
||||
|
||||
# Convert to pixel coords
|
||||
r0, c0 = src.index(left, top)
|
||||
r1, c1 = src.index(right, bottom)
|
||||
|
||||
row_min, row_max = sorted((r0, r1))
|
||||
col_min, col_max = sorted((c0, c1))
|
||||
|
||||
window = clamp_window(
|
||||
Window(col_min, row_min, col_max - col_min, row_max - row_min),
|
||||
src
|
||||
)
|
||||
if window is None:
|
||||
return False
|
||||
|
||||
# Read terrain RGB
|
||||
data = src.read(
|
||||
indexes=(1,2,3),
|
||||
out_shape=(3, 256, 256),
|
||||
window=window,
|
||||
resampling=Resampling.nearest
|
||||
)
|
||||
|
||||
if data.size == 0:
|
||||
return False
|
||||
|
||||
# Save tile
|
||||
out_dir = os.path.join(tiles_root, str(z), str(x))
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
out_path = os.path.join(out_dir, f"{y}.png")
|
||||
|
||||
Image.fromarray(data.transpose(1,2,0).astype("uint8"), "RGB").save(out_path)
|
||||
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ---------- Main ----------
|
||||
def main():
|
||||
tif_path = sys.argv[1]
|
||||
tiles_root = sys.argv[2]
|
||||
z_min = int(sys.argv[3])
|
||||
z_max = int(sys.argv[4])
|
||||
max_workers = int(sys.argv[5])
|
||||
|
||||
with rasterio.open(tif_path) as src:
|
||||
if src.crs is None:
|
||||
print("[WARN] CRS mancante:", tif_path)
|
||||
return
|
||||
|
||||
left, bottom, right, top = transform_bounds(src.crs, "EPSG:4326", *src.bounds)
|
||||
|
||||
jobs = []
|
||||
for z in range(z_min, z_max + 1):
|
||||
x_min, y_max = latlon_to_tile(bottom, left, z)
|
||||
x_max, y_min = latlon_to_tile(top, right, z)
|
||||
|
||||
x0, x1 = sorted((x_min, x_max))
|
||||
y0, y1 = sorted((y_min, y_max))
|
||||
|
||||
for x in range(x0, x1 + 1):
|
||||
for y in range(y0, y1 + 1):
|
||||
jobs.append((tif_path, tiles_root, z, x, y))
|
||||
|
||||
if not jobs:
|
||||
print("[WARN] Nessuna tile da generare per", tif_path)
|
||||
return
|
||||
|
||||
with ProcessPoolExecutor(max_workers=max_workers) as ex:
|
||||
for _ in ex.map(process_tile, jobs):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
EOF
|
||||
|
||||
|
||||
###############################################
|
||||
# LOOP SU TUTTI I TERRAINRGB (SOLO DEM)
|
||||
###############################################
|
||||
|
||||
for DEM_DIR in "$TERRAIN_ROOT"/*; do
|
||||
[ -d "$DEM_DIR" ] || continue
|
||||
|
||||
BASENAME=$(basename "$DEM_DIR")
|
||||
|
||||
# Processa SOLO DEM (salta EDM e FLM)
|
||||
case "$BASENAME" in
|
||||
*DEM) ;;
|
||||
*) echo "[SKIP] Non DEM: $BASENAME"; continue ;;
|
||||
esac
|
||||
|
||||
TIF="$DEM_DIR/${BASENAME}_terrain_rgb.tif"
|
||||
|
||||
if [ ! -f "$TIF" ]; then
|
||||
echo "[WARN] Nessun TIFF TerrainRGB in $DEM_DIR"
|
||||
continue
|
||||
fi
|
||||
|
||||
OUT_DIR="$TILES_ROOT/$BASENAME"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
RESUME_FILE="$RESUME_DIR/${BASENAME}.done"
|
||||
LOGFILE="$LOG_DIR/${BASENAME}.log"
|
||||
|
||||
if [ -f "$RESUME_FILE" ]; then
|
||||
echo "[SKIP] Tiles gi generati per $BASENAME"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[INFO] Genero tiles per $BASENAME"
|
||||
|
||||
{
|
||||
echo "----------------------------------------"
|
||||
echo "DEM: $BASENAME"
|
||||
echo "Input: $TIF"
|
||||
echo "Output: $OUT_DIR"
|
||||
echo "Zoom: $Z_MIN$Z_MAX"
|
||||
echo "Start: $(date)"
|
||||
echo "----------------------------------------"
|
||||
|
||||
CORES=$(sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
||||
|
||||
"$PYTHON" "$PY_SCRIPT" "$TIF" "$OUT_DIR" "$Z_MIN" "$Z_MAX" "$CORES"
|
||||
STATUS=$?
|
||||
|
||||
echo "End: $(date)"
|
||||
echo "Status: $STATUS"
|
||||
echo "----------------------------------------"
|
||||
|
||||
} >> "$LOGFILE" 2>&1
|
||||
|
||||
|
||||
if [ "$STATUS" -ne 0 ]; then
|
||||
echo "[ERRORE] Tiles falliti per $BASENAME"
|
||||
continue
|
||||
fi
|
||||
|
||||
touch "$RESUME_FILE"
|
||||
echo "[OK] Tiles creati per $BASENAME"
|
||||
|
||||
done
|
||||
|
||||
rm -f "$PY_SCRIPT"
|
||||
echo "[INFO] Fase 2 completata"
|
||||
|
||||
48
terrain.txt
Normal file
48
terrain.txt
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
🚀 Scaricare i dati da mirror Copernicus GLO‑30
|
||||
|
||||
aws s3 sync s3://copernicus-dem-30m ./glo30 --no-sign-request
|
||||
|
||||
Crea glo30 dir con tutto il mondo
|
||||
|
||||
1️⃣ Crea un terrain_rgb partendo da glo30
|
||||
|
||||
Utilizza ph1.sh
|
||||
|
||||
Puoi interrompere quando vuoi poi riparte tralasciano quello fatto (ci mette molto 1/2 gg)
|
||||
|
||||
Circa 600GB
|
||||
|
||||
Crea una dir terrain_rgb
|
||||
|
||||
|
||||
2️⃣ Crea i tiles partendo da terrain_rgb
|
||||
|
||||
Utilizza ph2.sh
|
||||
|
||||
Puoi interrompere quando vuoi poi riparte tralasciano quello fatto (ci mette moltissimo 1/2 weeks)
|
||||
|
||||
Circa 1.6TB
|
||||
|
||||
Crea una dir tiles
|
||||
|
||||
3️⃣ Estrarre soltanto una parte(opzionale)
|
||||
|
||||
Utilizza make_tiles_italy.sh
|
||||
|
||||
partendo da tiles ci mette solo i tiles dell'Italia
|
||||
|
||||
crea una di /Volumes/SSD4T/tiles_italy_new
|
||||
|
||||
4️⃣ Trasforma la dir tiles_italy_new in modo che sia utilizzabile damb-util
|
||||
|
||||
Utilizza merge_tiles.sh
|
||||
|
||||
crea una dir merged_tiles che può essere utilizzata con mb-util
|
||||
|
||||
5️⃣
|
||||
|
||||
|
||||
6️⃣
|
||||
|
||||
🎯 Risultato
|
||||
Hai un terrain globale ad alta qualità, basato su Copernicus GLO‑30, pronto per TileServer‑GL.
|
||||
Loading…
Reference in a new issue