iT邦幫忙

0

關於json 一問

大家好, 請問一下

因為我有一個>800000 以上支record的json,

{
    "type": "FeatureCollection",
    "features": [
        {
            "type": "Feature",
            "properties": {
                "olrtoa": 0.0,
                "rainc": 0.0,
                "rainnc": 0.0,
                "refl10cm_max": 0.0,
                "refl10cm_1km": 0.0,
                "refl10cm_1km_max": 0.0,
                "precipw": 41.7464,
                "u10": 1.4652,
                "v10": -4.3075,
                "q2": 0.0078,
                "t2m": 284.3044,
                "th2m": 0.0,
                "mslp": 102293.0781,
                "relhum_200hPa": 90.9454,
                "relhum_250hPa": 84.6387,
                "relhum_500hPa": 74.2758,
                "relhum_700hPa": 99.5619,
                "relhum_850hPa": 98.9876,
                "relhum_925hPa": 99.7717,
                "dewpoint_200hPa": 216.5222,
                "dewpoint_250hPa": 228.7396,
                "dewpoint_500hPa": 264.3343,
                "dewpoint_700hPa": 279.7273,
                "dewpoint_850hPa": 278.9214,
                "dewpoint_925hPa": 280.4701,
                "temperature_200hPa": 221.3499,
                "temperature_250hPa": 234.0024,
                "temperature_500hPa": 268.5225,
                "temperature_700hPa": 279.7969,
                "temperature_850hPa": 279.0697,
                "temperature_925hPa": 280.4991,
                "height_200hPa": 12497.9375,
                "height_250hPa": 11003.0938,
                "height_500hPa": 5862.3799,
                "height_700hPa": 3138.8628,
                "height_850hPa": 1535.4905,
                "height_925hPa": 838.147,
                "uzonal_200hPa": 18.3141,
                "uzonal_250hPa": 20.1415,
                "uzonal_500hPa": 23.2047,
                "uzonal_700hPa": 15.306,
                "uzonal_850hPa": -0.7148,
                "uzonal_925hPa": -1.9181,
                "umeridional_200hPa": -13.0623,
                "umeridional_250hPa": -6.7274,
                "umeridional_500hPa": 2.9739,
                "umeridional_700hPa": 5.5618,
                "umeridional_850hPa": -7.1236,
                "umeridional_925hPa": -10.9511,
                "w_200hPa": 0.0016,
                "w_250hPa": 0.0047,
                "w_500hPa": 0.0208,
                "w_700hPa": 0.0107,
                "w_850hPa": -0.0113,
                "w_925hPa": -0.0152,
                "t_isobaric": [
                    244.6622,
                    253.3894,
                    259.8877,
                    264.8364,
                    268.5225
                ],
                "z_isobaric": [
                    9729.2197,
                    8599.5508,
                    7590.7847,
                    6683.9741,
                    5862.3799,
                    5109.5322,
                    4409.6255,
                    3753.0161,
                    3138.8628,
                    2566.4587,
                    2033.6993,
                    1535.4904,
                    1065.0464
                ],
                "meanT_500_300": 258.9046,
                "cape": 0.0,
                "cin": 0.0,
                "lcl": 0.0,
                "lfc": 0.0,
                "srh_0_1km": 48.0264,
                "srh_0_3km": 313.8358,
                "uzonal_surface": 2.1873,
                "uzonal_1km": -2.2103,
                "uzonal_6km": 23.1599,
                "umeridional_surface": -5.8998,
                "umeridional_1km": -9.4549,
                "umeridional_6km": 2.4352,
                "temperature_surface": 284.1897,
                "dewpoint_surface": 283.6391,
                "updraft_helicity_max": 0.0006,
                "w_velocity_max": 0.0245,
                "wind_speed_level1_max": 6.2922,
                "t_oml": 0.0,
                "t_oml_initial": 0.0,
                "t_oml_200m_initial": 0.0,
                "h_oml": 0.0,
                "h_oml_initial": 0.0,
                "hu_oml": 0.0,
                "hv_oml": 0.0,
                "pressure": [
                    101013.7969,
                    100164.4453,
                    99045.7891,
                    97560.1406,
                    95619.0078,
                    93148.2266,
                    90091.7578,
                    86415.5156,
                    82115.7188,
                    77229.1641,
                    71824.0078,
                    65964.7812,
                    59727.3359,
                    53513.957,
                    47768.1602,
                    42563.5977,
                    37842.6172,
                    33555.6094,
                    29657.8438,
                    26114.8555,
                    22901.918,
                    19994.6934,
                    17376.6211,
                    15030.8252,
                    12947.9326,
                    11116.5762,
                    9518.2783,
                    8147.874,
                    6986.5503,
                    6002.7388,
                    5169.4463,
                    4462.0708,
                    3859.8184,
                    3345.1638,
                    2903.7214,
                    2523.5347,
                    2195.083,
                    1911.0852,
                    1665.2407,
                    1452.1255,
                    1267.0046
                ],
                "divergence": [
                    0.0,
                    0.0,
                    0.0,
                    0.0,
                    -0.0,
                    -0.0,
                    0.0,
                    0.0,
                    -0.0,
                    -0.0001,
                    -0.0001,
                    -0.0001,
                    0.0001,
                    0.0001,
                    0.0,
                    0.0,
                    0.0,
                    0.0,
                    -0.0,
                    -0.0,
                    0.0,
                    0.0,
                    -0.0,
                    -0.0,
                    -0.0,
                    -0.0,
                    0.0,
                    -0.0,
                    -0.0,
                    -0.0,
                    0.0,
                    -0.0,
                    -0.0,
                    0.0,
                    0.0,
                    0.0,
                    0.0,
                    -0.0,
                    -0.0,
                    -0.0,
                    -0.0
                ],
                "uReconstructZonal": [
                    2.1873,
                    2.1616,
                    2.1274,
                    2.0801,
                    0.3551,
                    -1.8267,
                    -2.2578,
                    -1.5089,
                    0.9033,
                    5.8838,
                    13.0326,
                    20.3354,
                    22.8974,
                    23.3174,
                    23.1331,
                    20.1204,
                    20.5513,
                    22.6693,
                    23.1613,
                    20.6357,
                    19.2114,
                    18.3125,
                    15.3522,
                    12.3908,
                    11.6806,
                    10.9877,
                    9.3545,
                    5.9802,
                    2.9038,
                    3.5083,
                    4.1131,
                    4.3958,
                    5.5911,
                    8.9163,
                    11.0295,
                    10.2801,
                    9.5307,
                    8.8073,
                    8.1224,
                    7.8082,
                    8.294
                ],
                "uReconstructMeridional": [
                    -5.8998,
                    -6.968,
                    -8.3858,
                    -10.2893,
                    -11.69,
                    -11.3256,
                    -9.56,
                    -7.9052,
                    -5.531,
                    -0.5967,
                    5.4521,
                    5.8045,
                    4.3929,
                    4.3274,
                    2.1142,
                    -1.2393,
                    -2.233,
                    -2.9979,
                    -6.7025,
                    -5.7705,
                    -8.5281,
                    -13.0706,
                    -13.4715,
                    -13.8725,
                    -9.2893,
                    -4.6641,
                    -2.1546,
                    -3.565,
                    -4.8218,
                    -4.1828,
                    -3.5436,
                    -2.587,
                    -0.842,
                    2.4907,
                    4.4148,
                    3.0124,
                    1.6099,
                    0.2652,
                    -0.9938,
                    -1.6833,
                    -1.1434
                ],
                "latCell": 0.4839,
                "lonCell": 1.9676,
                "nEdgesOnCell": 6.0,
                "edgesOnCell": [
                    1,
                    3,
                    5,
                    7,
                    9,
                    11,
                    0
                ],
                "areaCell": 9780523.0,
                "cellsOnCell": [
                    734701,
                    735074,
                    735073,
                    734287,
                    733079,
                    733503,
                    0
                ],
                "verticesOnCell": [
                    1,
                    2,
                    3,
                    4,
                    5,
                    6,
                    0
                ],
                "meshDensity": 0.6561,
                "zgrid": [
                    75.7157,
                    136.9278,
                    216.7624,
                    324.1465,
                    467.9984,
                    657.2205,
                    900.6932,
                    1207.2616,
                    1585.7206,
                    2044.8031,
                    2593.248,
                    3240.0247,
                    3994.4692,
                    4860.6265,
                    5757.4414,
                    6654.0981,
                    7550.436,
                    8446.541,
                    9342.4824,
                    10238.1611,
                    11133.4307,
                    12028.5615,
                    12923.8008,
                    13819.3418,
                    14715.3105,
                    15611.79,
                    16508.8125,
                    17406.373,
                    18304.4434,
                    19202.9746,
                    20101.8965,
                    21001.1465,
                    21900.6465,
                    22800.3359,
                    23700.1602,
                    24600.0664,
                    25500.0234,
                    26400.0059,
                    27300.0,
                    28200.0,
                    29100.0,
                    30000.0
                ],
                "uzonal_300hPa": 23.1181,
                "umeridional_300hPa": -6.3773,
                "gph_200hPa": 12450.1699,
                "gph_300hPa": 9696.8945,
                "gph_500hPa": 5852.041,
                "lat": [
                    27.7099,
                    27.7077,
                    27.7217,
                    27.7388,
                    27.741,
                    27.7269
                ],
                "lon": [
                    112.7265,
                    112.7436,
                    112.7571,
                    112.7487,
                    112.7315,
                    112.718
                ],
                "latlon": [
                    [
                        112.7265,
                        27.7099
                    ],
                    [
                        112.7436,
                        27.7077
                    ],
                    [
                        112.7571,
                        27.7217
                    ],
                    [
                        112.7487,
                        27.7388
                    ],
                    [
                        112.7315,
                        27.741
                    ],
                    [
                        112.718,
                        27.7269
                    ]
                ],
                "bbox": [
                    27.7077,
                    112.718,
                    27.741,
                    112.7571
                ],
                "location": "X021, Shitan, Xiangtan County, Xiangtan, Hunan, China"
            },
            "geometry": {
                "type": "Polygon",
                "coordinates": [
                    [
                        [
                            112.7265,
                            27.7099
                        ],
                        [
                            112.718,
                            27.7269
                        ],
                        [
                            112.7315,
                            27.741
                        ],
                        [
                            112.7487,
                            27.7388
                        ],
                        [
                            112.7571,
                            27.7217
                        ],
                        [
                            112.7436,
                            27.7077
                        ],
                        [
                            112.7265,
                            27.7099
                        ]
                    ]
                ]
            },
            "bbox": [
                27.7077,
                112.718,
                27.741,
                112.7571
            ]
        },
        {
            "type": "Feature",
            "properties": {
            ..
            ..
            ..
            ..

Feature 就是重覆的項目,

請問一下如果把那些數據抽出, 把那些重覆的type properties 寫成一個template json

{
    "type":"FeatureCollection",
    "features":[
        {
            "type":"Feature",
            "properties":{
			},
			"geometry":{
			},
			"bbox":[
			]
		}
	],
	"bbox":[
        -180,
        -90,
        180,
        83.64513
    ]
}

到底是在js 那邊load 一個完整的json , 還是分開一個template, 再把數據在js寫入再構成一個完整json 比較快, 謝謝.

1 個回答

1
海綿寶寶
iT邦大神 1 級 ‧ 2021-11-05 15:54:32

這問題
之前 不是已經解決了嗎?

在繼續之前
先來算個數學
用上面的 sample data 來計算
每筆 feature 約有 14,345 bytes
乘上 800,000 筆
14,345 x 800,000 = 11,476,000,000,大概是 11G

與其研究「一次讀 json」或是「用 template 組 json」那個快
我建議你先想一下
有沒有其他更好的方式來處理這 11G 的資料

看更多先前的回應...收起先前的回應...
diu7me iT邦新手 4 級 ‧ 2021-11-05 17:01:07 檢舉

你好, 海綿寶寶, 其實那個json 是已經gzip 了...之後再在JS 那邊ungzip

diu7me iT邦新手 4 級 ‧ 2021-11-05 17:02:25 檢舉

5000支=> 6mb左右

diu7me iT邦新手 4 級 ‧ 2021-11-05 17:06:13 檢舉

800000支只是1GB左右

淺水員 iT邦高手 2 級 ‧ 2021-11-05 19:37:44 檢舉

有必要一次讀取嗎?
只讀取目前必要的部分會不會比較好?

diu7me iT邦新手 4 級 ‧ 2021-11-06 14:27:59 檢舉

因為globe.gl要那些數據來畫那個地球上的polygon, 他也沒有什麼方法是可以drag 那個地球的時候, load 那些看到的部分

diu7me iT邦新手 4 級 ‧ 2021-11-16 14:26:56 檢舉

你好, 屠豬士, 想問一下那如果我的geojson feature 是polygon , 不是point 呢? 那還可以用spatial 嗎?

我要發表回答

立即登入回答