Skip to content

Commit 4767259

Browse files
elifsu-simulamartinRenou
elifsu-simula
and
martinRenou
authored
Add GeoParquet support (#727)
* Add Geoparquet source from URL * Add Geoparquet source from file * Add Geoparquet source to Python API * Add Geoparquet source from URL * Add Geoparquet source from file * Add Geoparquet source to Python API * Update yarn.lock * Add parquet file * Move dependencies to to packages/base * Format files * Fix icon * Add example project --------- Co-authored-by: martinRenou <[email protected]>
1 parent e16eb57 commit 4767259

File tree

18 files changed

+303
-2
lines changed

18 files changed

+303
-2
lines changed

‎examples/geoparquet.jgis

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{
2+
"layerTree": [
3+
"9556ca29-a5ec-41af-bf14-4b543c52aafe",
4+
"d7a2ad84-0750-4e9b-82c0-542fcc6b3265"
5+
],
6+
"layers": {
7+
"9556ca29-a5ec-41af-bf14-4b543c52aafe": {
8+
"name": "OpenStreetMap.Mapnik Layer",
9+
"parameters": {
10+
"source": "2a52082b-7992-40dc-92d6-75e309a1ed27"
11+
},
12+
"type": "RasterLayer",
13+
"visible": true
14+
},
15+
"d7a2ad84-0750-4e9b-82c0-542fcc6b3265": {
16+
"name": "Custom GeoParquet Layer",
17+
"parameters": {
18+
"opacity": 1.0,
19+
"source": "c1da95b9-8a71-4fee-b4e3-6f0b5f53d2d4"
20+
},
21+
"type": "VectorLayer",
22+
"visible": true
23+
}
24+
},
25+
"metadata": {},
26+
"options": {
27+
"bearing": 0.0,
28+
"extent": [
29+
-20037508.342789244,
30+
-15214174.147482341,
31+
20037508.342789244,
32+
15214174.147482341
33+
],
34+
"latitude": 0.0,
35+
"longitude": 0.0,
36+
"projection": "EPSG:3857",
37+
"zoom": 1.5058115539195944
38+
},
39+
"schemaVersion": "0.5.0",
40+
"sources": {
41+
"2a52082b-7992-40dc-92d6-75e309a1ed27": {
42+
"name": "OpenStreetMap.Mapnik",
43+
"parameters": {
44+
"attribution": "(C) OpenStreetMap contributors",
45+
"maxZoom": 19.0,
46+
"minZoom": 0.0,
47+
"provider": "OpenStreetMap",
48+
"url": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
49+
"urlParameters": {}
50+
},
51+
"type": "RasterSource"
52+
},
53+
"c1da95b9-8a71-4fee-b4e3-6f0b5f53d2d4": {
54+
"name": "Custom GeoParquet Source",
55+
"parameters": {
56+
"attribution": "",
57+
"path": "https://raw.githubusercontent.com/opengeospatial/geoparquet/main/examples/example.parquet",
58+
"projection": "EPSG:4326"
59+
},
60+
"type": "GeoParquetSource"
61+
}
62+
}
63+
}

‎packages/base/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,9 @@
8080
"date-fns": "^4.1.0",
8181
"gdal3.js": "^2.8.1",
8282
"geojson-vt": "^4.0.2",
83+
"geoparquet": "^0.3.0",
8384
"geotiff": "^2.1.3",
85+
"hyparquet-compressors": "^1.1.1",
8486
"lucide-react": "^0.513.0",
8587
"ol": "^10.1.0",
8688
"ol-pmtiles": "^0.5.0",

‎packages/base/src/commands/BaseCommandIDs.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ export const newHillshadeEntry = 'jupytergis:newHillshadeEntry';
2424
export const newImageEntry = 'jupytergis:newImageEntry';
2525
export const newVideoEntry = 'jupytergis:newVideoEntry';
2626
export const newGeoTiffEntry = 'jupytergis:newGeoTiffEntry';
27+
export const newGeoParquetEntry = 'jupytergis:newGeoParquetEntry';
2728

2829
// Layer and group actions
2930
export const renameLayer = 'jupytergis:renameLayer';

‎packages/base/src/commands/index.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,27 @@ export function addCommands(
320320
...icons.get(CommandIDs.newVectorTileEntry),
321321
});
322322

323+
commands.addCommand(CommandIDs.newGeoParquetEntry, {
324+
label: trans.__('New GeoParquet Layer'),
325+
isEnabled: () => {
326+
return tracker.currentWidget
327+
? tracker.currentWidget.model.sharedModel.editable
328+
: false;
329+
},
330+
execute: Private.createEntry({
331+
tracker,
332+
formSchemaRegistry,
333+
title: 'Create GeoParquet Layer',
334+
createLayer: true,
335+
createSource: true,
336+
sourceData: { name: 'Custom GeoParquet Source' },
337+
layerData: { name: 'Custom GeoParquet Layer' },
338+
sourceType: 'GeoParquetSource',
339+
layerType: 'VectorLayer',
340+
}),
341+
...icons.get(CommandIDs.newGeoParquetEntry),
342+
});
343+
323344
commands.addCommand(CommandIDs.newGeoJSONEntry, {
324345
label: trans.__('New GeoJSON layer'),
325346
isEnabled: () => {

‎packages/base/src/constants.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ const iconObject = {
5252
[CommandIDs.newVideoEntry]: { iconClass: 'fa fa-video' },
5353
[CommandIDs.newShapefileEntry]: { iconClass: 'fa fa-file' },
5454
[CommandIDs.newGeoTiffEntry]: { iconClass: 'fa fa-image' },
55+
[CommandIDs.newGeoParquetEntry]: { iconClass: 'fa fa-file' },
5556
[CommandIDs.symbology]: { iconClass: 'fa fa-brush' },
5657
[CommandIDs.identify]: { icon: infoIcon },
5758
[CommandIDs.temporalController]: { icon: clockIcon },

‎packages/base/src/formbuilder/formselectors.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ export function getSourceTypeForm(
6060
case 'VectorTileSource':
6161
SourceForm = TileSourcePropertiesForm;
6262
break;
63+
case 'GeoParquetSource':
64+
SourceForm = PathBasedSourcePropertiesForm;
65+
break;
66+
6367
// ADD MORE FORM TYPES HERE
6468
}
6569
return SourceForm;

‎packages/base/src/mainview/mainView.tsx

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
IVectorLayer,
2727
IVectorTileLayer,
2828
IVectorTileSource,
29+
IGeoParquetSource,
2930
IWebGlLayer,
3031
JgisCoordinates,
3132
JupyterGISModel,
@@ -769,6 +770,28 @@ export class MainView extends React.Component<IProps, IStates> {
769770

770771
break;
771772
}
773+
774+
case 'GeoParquetSource': {
775+
const parameters = source.parameters as IGeoParquetSource;
776+
777+
const geojson = await loadFile({
778+
filepath: parameters.path,
779+
type: 'GeoParquetSource',
780+
model: this._model,
781+
});
782+
783+
const geojsonData = Array.isArray(geojson) ? geojson[0] : geojson;
784+
785+
const format = new GeoJSON();
786+
787+
newSource = new VectorSource({
788+
features: format.readFeatures(geojsonData, {
789+
dataProjection: parameters.projection,
790+
featureProjection: this._Map.getView().getProjection(),
791+
}),
792+
});
793+
break;
794+
}
772795
}
773796

774797
newSource.set('id', id);

‎packages/base/src/menus.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ export const vectorSubMenu = (commands: CommandRegistry) => {
2626
command: CommandIDs.newShapefileEntry,
2727
});
2828

29+
subMenu.addItem({
30+
type: 'command',
31+
command: CommandIDs.newGeoParquetEntry,
32+
});
33+
2934
return subMenu;
3035
};
3136

‎packages/base/src/tools.ts

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { PathExt, URLExt } from '@jupyterlab/coreutils';
1212
import { Contents, ServerConnection } from '@jupyterlab/services';
1313
import { VectorTile } from '@mapbox/vector-tile';
1414
import * as d3Color from 'd3-color';
15+
import { compressors } from 'hyparquet-compressors';
1516
import Protobuf from 'pbf';
1617
import shp from 'shpjs';
1718

@@ -567,6 +568,26 @@ export const loadFile = async (fileInfo: {
567568
throw new Error(`Failed to fetch ${filepath}`);
568569
}
569570

571+
case 'GeoParquetSource': {
572+
const cached = await getFromIndexedDB(filepath);
573+
if (cached) {
574+
return cached.file;
575+
}
576+
577+
const { asyncBufferFromUrl, toGeoJson } = await import('geoparquet');
578+
579+
const file = await asyncBufferFromUrl({ url: filepath });
580+
const geojson = await toGeoJson({ file });
581+
582+
if (geojson) {
583+
await saveToIndexedDB(filepath, geojson);
584+
return geojson;
585+
}
586+
587+
showErrorMessage('Network error', `Failed to fetch ${filepath}`);
588+
throw new Error(`Failed to fetch ${filepath}`);
589+
}
590+
570591
default: {
571592
throw new Error(`Unsupported URL handling for source type: ${type}`);
572593
}
@@ -633,6 +654,18 @@ export const loadFile = async (fileInfo: {
633654
}
634655
}
635656

657+
case 'GeoParquetSource': {
658+
if (typeof file.content === 'string') {
659+
const { toGeoJson } = await import('geoparquet');
660+
661+
const arrayBuffer = await stringToArrayBuffer(file.content as string);
662+
663+
return await toGeoJson({ file: arrayBuffer, compressors });
664+
} else {
665+
throw new Error('Invalid file format for GeoParquet content.');
666+
}
667+
}
668+
636669
default: {
637670
throw new Error(`Unsupported source type: ${type}`);
638671
}

‎packages/base/src/types/geoparquet.d.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
declare module 'geoparquet' {
2+
export function asyncBufferFromUrl(options: {
3+
url: string;
4+
byteLength?: number;
5+
requestInit?: RequestInit;
6+
}): Promise<AsyncBuffer>;
7+
8+
export function toGeoJson(options: {
9+
file: AsyncBuffer;
10+
compressors?: any;
11+
}): Promise<GeoJSON>;
12+
};

0 commit comments

Comments
 (0)