From c7786f03ecb410d059daa683b1a04722896ab5e0 Mon Sep 17 00:00:00 2001 From: afischerdev Date: Fri, 27 Oct 2023 16:22:52 +0200 Subject: [PATCH] added 1sec bef generation --- .../btools/mapcreator/ConvertLidarTile.java | 276 +++++++++++++++--- 1 file changed, 236 insertions(+), 40 deletions(-) diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/ConvertLidarTile.java b/brouter-map-creator/src/main/java/btools/mapcreator/ConvertLidarTile.java index 2a00e96..cab1d0b 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/ConvertLidarTile.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/ConvertLidarTile.java @@ -9,13 +9,15 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; public class ConvertLidarTile { - private static int NROWS; - private static int NCOLS; + + public static final boolean DEBUG = false; public static final short NODATA2 = -32767; // hgt-formats nodata public static final short NODATA = Short.MIN_VALUE; @@ -25,17 +27,89 @@ public class ConvertLidarTile { private static final int HGT_3ASEC_ROWS = 1201; // 3 arc second resolution (90m) private static final int HGT_3ASEC_FILE_SIZE = HGT_3ASEC_ROWS * HGT_3ASEC_ROWS * Short.BYTES; private static final int HGT_1ASEC_ROWS = 3601; // 1 arc second resolution (30m) + private static final int SRTM3_ROW_LENGTH = 1200; // number of elevation values per line + private static final int SRTM1_ROW_LENGTH = 3600; + private static final boolean SRTM_NO_ZERO = true; - static short[] imagePixels; + private int NROWS; + private int NCOLS; + private int ROW_LENGTH; + private short[] imagePixels; - private static void readHgtZip(String filename, int rowOffset, int colOffset) throws Exception { + + public static void main(String[] args) throws Exception { + if (args.length == 3 || args.length == 4 || args.length == 5) { + String filename90 = args[0]; + if ("all".equals(filename90)) { + //if (DEBUG) + System.out.println("lidar convert all "); + new ConvertLidarTile().doConvertAll(args[1], args[2], (args.length > 3 ? args[3] : null), (args.length == 5 ? args[4] : null)); + return; + } + // old filenames only + String filename30 = filename90 + ".bef"; //filename90.substring(0, filename90.length() - 3) + "bef"; + + int srtmLonIdx = Integer.parseInt(filename90.substring(5, 7).toLowerCase()); + int srtmLatIdx = Integer.parseInt(filename90.substring(8, 10).toLowerCase()); + + int ilon_base = (srtmLonIdx - 1) * 5 - 180; + int ilat_base = 150 - srtmLatIdx * 5 - 90; + int row_length = SRTM3_ROW_LENGTH; + String fallbackdir = null; + if (args.length > 3) { + row_length = (Integer.parseInt(args[3]) == 1 ? SRTM1_ROW_LENGTH : SRTM3_ROW_LENGTH); + fallbackdir = (args.length == 5 ? args[4] : null); + } + //if (DEBUG) + System.out.println("lidar convert " + ilon_base + " " + ilat_base + " from " + srtmLonIdx + " " + srtmLatIdx + " f: " + filename90 + " rowl " + row_length); + + new ConvertLidarTile().doConvert(args[1], ilon_base, ilat_base, args[2] + "/" + filename30, row_length, fallbackdir); + } else { + System.out.println("usage: java [arc seconds (1 or 3,default=3)] [hgt-fallback-data-dir]"); + System.out.println("or java all [arc seconds (1 or 3, default=3)] [hgt-fallback-data-dir]"); + return; + } + } + + private void doConvertAll(String hgtdata, String outdir, String rlen, String hgtfallbackdata) throws Exception { + int row_length = SRTM3_ROW_LENGTH; + if (rlen != null) { + row_length = (Integer.parseInt(rlen) == 1 ? SRTM1_ROW_LENGTH : SRTM3_ROW_LENGTH); + } + String filename30; + for (int ilon_base = -180; ilon_base < 180; ilon_base += 5) { + for (int ilat_base = 85; ilat_base > -90; ilat_base -= 5) { + if (PosUnifier.UseLidarRd5FileName) { + filename30 = genFilenameRd5(ilon_base, ilat_base); + } else { + filename30 = genFilenameOld(ilon_base, ilat_base); + } + if (DEBUG) + System.out.println("lidar convert all: " + filename30); + doConvert(hgtdata, ilon_base, ilat_base, outdir + "/" + filename30, row_length, hgtfallbackdata); + } + } + } + + static String genFilenameOld(int ilon_base, int ilat_base) { + int srtmLonIdx = ((ilon_base + 180) / 5) + 1; + int srtmLatIdx = (60 - ilat_base) / 5; + return String.format("srtm_%02d_%02d.bef", srtmLonIdx, srtmLatIdx); + } + + static String genFilenameRd5(int ilon_base, int ilat_base) { + return String.format("srtm_%s_%s.bef", ilon_base < 0 ? "W" + (-ilon_base) : "E" + ilon_base, + ilat_base < 0 ? "S" + (-ilat_base) : "N" + ilat_base); + } + + private void readHgtZip(String filename, int rowOffset, int colOffset, int row_length) throws Exception { ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(filename))); try { for (; ; ) { ZipEntry ze = zis.getNextEntry(); if (ze == null) break; if (ze.getName().toLowerCase().endsWith(HGT_FILE_EXT)) { - readHgtFromStream(zis, rowOffset, colOffset, HGT_3ASEC_ROWS); + readHgtFromStream(zis, rowOffset, colOffset, row_length, 1); return; } } @@ -44,14 +118,14 @@ public class ConvertLidarTile { } } - private static void readHgtFromStream(InputStream is, int rowOffset, int colOffset, int rowLength) + private void readHgtFromStream(InputStream is, int rowOffset, int colOffset, int rowLength, int scale) throws Exception { DataInputStream dis = new DataInputStream(new BufferedInputStream(is)); for (int ir = 0; ir < rowLength; ir++) { - int row = rowOffset + ir; + int row = rowOffset + ir * scale; for (int ic = 0; ic < rowLength; ic++) { - int col = colOffset + ic; + int col = colOffset + ic * scale; int i1 = dis.read(); // msb first! int i0 = dis.read(); @@ -64,19 +138,52 @@ public class ConvertLidarTile { if (val == NODATA2) { val = NODATA; } - - setPixel(row, col, val); + if (scale == 3) { + setPixel(row, col, val); + setPixel(row + 1, col, val); + setPixel(row + 2, col, val); + setPixel(row, col + 1, val); + setPixel(row + 1, col + 1, val); + setPixel(row + 2, col + 1, val); + setPixel(row, col + 2, val); + setPixel(row + 1, col + 2, val); + setPixel(row + 2, col + 2, val); + } else { + setPixel(row, col, val); + } } } } - private static void setPixel(int row, int col, short val) { + private void readFallbackFile(File file, int rowOffset, int colOffset, int row_length) + throws Exception { + int rowLength; + int scale; + if (file.length() > HGT_3ASEC_FILE_SIZE) { + rowLength = HGT_1ASEC_ROWS; + scale = 1; + } else { + rowLength = HGT_3ASEC_ROWS; + scale = 3; + } + if (DEBUG) + System.out.println("read fallback: " + file + " " + rowLength); + + FileInputStream fis = new FileInputStream(file); + try { + readHgtFromStream(fis, rowOffset, colOffset, rowLength, scale); + } finally { + fis.close(); + } + } + + private void setPixel(int row, int col, short val) { if (row >= 0 && row < NROWS && col >= 0 && col < NCOLS) { imagePixels[row * NCOLS + col] = val; } } - private static short getPixel(int row, int col) { + private short getPixel(int row, int col) { if (row >= 0 && row < NROWS && col >= 0 && col < NCOLS) { return imagePixels[row * NCOLS + col]; } @@ -84,36 +191,137 @@ public class ConvertLidarTile { } - public static void doConvert(String inputDir, int lonDegreeStart, int latDegreeStart, String outputFile) throws Exception { + public void doConvert(String inputDir, int lonDegreeStart, int latDegreeStart, String outputFile, int row_length, String hgtfallbackdata) throws Exception { int extraBorder = 0; - NROWS = 5 * 1200 + 1 + 2 * extraBorder; - NCOLS = 5 * 1200 + 1 + 2 * extraBorder; + List foundList = new ArrayList<>(); + List notfoundList = new ArrayList<>(); - imagePixels = new short[NROWS * NCOLS]; // 650 MB ! + boolean found = false; - // prefill as NODATA - for (int row = 0; row < NROWS; row++) { - for (int col = 0; col < NCOLS; col++) { - imagePixels[row * NCOLS + col] = NODATA; + if (row_length == SRTM1_ROW_LENGTH) { + // check for sources w/o border + for (int latIdx = 0; latIdx < 5; latIdx++) { + int latDegree = latDegreeStart + latIdx; + + for (int lonIdx = 0; lonIdx < 5; lonIdx++) { + int lonDegree = lonDegreeStart + lonIdx; + + String filename = inputDir + "/" + formatLat(latDegree) + formatLon(lonDegree) + ".zip"; + File f = new File(filename); + if (f.exists() && f.length() > 0) { + found = true; + break; + } + } } + } else { + // ignore when srtm3 + found = true; + } + if (found) { // init when found + NROWS = 5 * row_length + 1 + 2 * extraBorder; + NCOLS = 5 * row_length + 1 + 2 * extraBorder; + imagePixels = new short[NROWS * NCOLS]; // 650 MB ! + + // prefill as NODATA + Arrays.fill(imagePixels, NODATA); + } else { + if (DEBUG) + System.out.println("none 1sec data: " + lonDegreeStart + " " + latDegreeStart); + return; } for (int latIdx = -1; latIdx <= 5; latIdx++) { int latDegree = latDegreeStart + latIdx; - int rowOffset = extraBorder + (4 - latIdx) * 1200; + int rowOffset = extraBorder + (4 - latIdx) * row_length; for (int lonIdx = -1; lonIdx <= 5; lonIdx++) { int lonDegree = lonDegreeStart + lonIdx; - int colOffset = extraBorder + lonIdx * 1200; + int colOffset = extraBorder + lonIdx * row_length; String filename = inputDir + "/" + formatLat(latDegree) + formatLon(lonDegree) + ".zip"; File f = new File(filename); if (f.exists() && f.length() > 0) { - System.out.println("exist: " + filename); - readHgtZip(filename, rowOffset, colOffset); + if (DEBUG) + System.out.println("exist: " + filename); + readHgtZip(filename, rowOffset, colOffset, row_length + 1); } else { - System.out.println("none : " + filename); + if (hgtfallbackdata != null) { + String filenamehgt = hgtfallbackdata + "/" + formatLat(latDegree) + formatLon(lonDegree) + ".hgt"; + f = new File(filenamehgt); + if (f.exists() && f.length() > 0) { + readFallbackFile(f, rowOffset, colOffset, row_length + 1); + /*if (imagePixels == null) { + imagePixels = new short[NROWS * NCOLS]; + Arrays.fill(imagePixels, NODATA); + found = true; + } + */ + /* + int rowLength; + int arcspace; + if (f.length() > HGT_3ASEC_FILE_SIZE) { + rowLength = HGT_1ASEC_ROWS; + arcspace = 1; + } else { + rowLength = HGT_3ASEC_ROWS; + arcspace = 3; + } + if (DEBUG) + System.out.println("read fallback: " + f + " " + rowLength); + + FileInputStream fis = new FileInputStream(f); + DataInputStream dis = new DataInputStream(new BufferedInputStream(fis)); + for (int ir = 0; ir < rowLength; ir++) { + int row = rowOffset + ir * arcspace; + + for (int ic = 0; ic < rowLength; ic++) { + int col = colOffset + ic * arcspace; + + int i1 = dis.read(); // msb first! + int i0 = dis.read(); + + if (i0 == -1 || i1 == -1) + throw new RuntimeException("unexpected end of file reading hgt entry!"); + + short val = (short) ((i1 << 8) | i0); + + if (val == NODATA2) { + val = NODATA; + } + if (arcspace == 3) { + setPixel(row, col, val); + setPixel(row+1, col, val); + setPixel(row+2, col, val); + setPixel(row, col+1, val); + setPixel(row+1, col+1, val); + setPixel(row+2, col+1, val); + setPixel(row, col+2, val); + setPixel(row+1, col+2, val); + setPixel(row+2, col+2, val); + } else { + setPixel(row, col, val); + } + } + } + fis.close(); + */ + } else { + if (DEBUG) + System.out.println("none : " + filename); + } + } + + } + } + } + + // post fill zero + if (SRTM_NO_ZERO) { + for (int row = 0; row < NROWS; row++) { + for (int col = 0; col < NCOLS; col++) { + if (imagePixels[row * NCOLS + col] == 0) imagePixels[row * NCOLS + col] = NODATA; } } } @@ -126,7 +334,7 @@ public class ConvertLidarTile { raster.ncols = NCOLS; raster.halfcol = halfCol5; raster.noDataValue = NODATA; - raster.cellsize = 1 / 1200.; + raster.cellsize = 1. / row_length; raster.xllcorner = lonDegreeStart - (0.5 + extraBorder) * raster.cellsize; raster.yllcorner = latDegreeStart - (0.5 + extraBorder) * raster.cellsize; raster.eval_array = imagePixels; @@ -156,6 +364,7 @@ public class ConvertLidarTile { } } } + imagePixels = null; } private static String formatLon(int lon) { @@ -181,19 +390,6 @@ public class ConvertLidarTile { return s + n.substring(n.length() - 2); } - public static void main(String[] args) throws Exception { - String filename90 = args[0]; - String filename30 = filename90.substring(0, filename90.length() - 3) + "bef"; - - int srtmLonIdx = Integer.parseInt(filename90.substring(5, 7).toLowerCase()); - int srtmLatIdx = Integer.parseInt(filename90.substring(8, 10).toLowerCase()); - - int ilon_base = (srtmLonIdx - 1) * 5 - 180; - int ilat_base = 150 - srtmLatIdx * 5 - 90; - - doConvert(args[1], ilon_base, ilat_base, filename30); - } - public SrtmRaster getRaster(File f, double lon, double lat) throws Exception { long fileSize; InputStream inputStream; @@ -231,7 +427,7 @@ public class ConvertLidarTile { // prefill as NODATA Arrays.fill(imagePixels, NODATA); - readHgtFromStream(inputStream, 0, 0, rowLength); + readHgtFromStream(inputStream, 0, 0, rowLength, 1); inputStream.close(); SrtmRaster raster = new SrtmRaster();