From 752cc7e19d22625436e55985288ecbf7b9fa5221 Mon Sep 17 00:00:00 2001 From: qdxkrs Date: Mon, 28 Feb 2022 13:51:30 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96flask?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- __pycache__/app.cpython-39.pyc | Bin 494 -> 0 bytes api/hello.py | 7 + app.py | 16 - application/__init__.py | 7 +- common/config/factory.py | 11 +- {scan_data => data_meta}/GetMetaInfo.py | 573 +++++++- {scan_data => data_meta}/__init__.py | 0 dms_client.db | Bin 36864 -> 40960 bytes requirements.txt | 14 +- run.py | 8 + .../__pycache__/GetMetaInfo.cpython-39.pyc | Bin 30380 -> 0 bytes scan_data/__pycache__/__init__.cpython-39.pyc | Bin 209 -> 0 bytes scan_data/__pycache__/example.cpython-39.pyc | Bin 3887 -> 0 bytes scan_data/example.py | 165 --- scan_data/receive_queue.py | 95 -- scheduled_task/queue_storage.py | 119 ++ upload/__init__.py | 5 - upload/__pycache__/__init__.cpython-39.pyc | Bin 205 -> 0 bytes .../__pycache__/upload_client.cpython-39.pyc | Bin 2021 -> 0 bytes util/UDP_Receive.py | 26 +- .../file_store_path.cpython-39.pyc | Bin 1650 -> 0 bytes .../http_file_upload.cpython-39.pyc | Bin 1955 -> 0 bytes .../remote_sensing_util.cpython-39.pyc | Bin 2803 -> 0 bytes .../__pycache__/scan_file_util.cpython-39.pyc | Bin 23608 -> 0 bytes util/file_store_path.py | 84 -- util/file_util.py | 1050 -------------- util/find_GPS_image.py | 81 -- util/http_file_upload.py | 86 -- util/remote_sensing_util.py | 91 -- util/scan_file_util.py | 1219 ----------------- util/simple_sqlite3_tool.py | 132 +- {upload => util}/upload_client.py | 46 +- util/zxby.py | 78 -- 33 files changed, 814 insertions(+), 3099 deletions(-) delete mode 100644 __pycache__/app.cpython-39.pyc create mode 100644 api/hello.py delete mode 100644 app.py rename {scan_data => data_meta}/GetMetaInfo.py (75%) rename {scan_data => data_meta}/__init__.py (100%) create mode 100644 run.py delete mode 100644 scan_data/__pycache__/GetMetaInfo.cpython-39.pyc delete mode 100644 scan_data/__pycache__/__init__.cpython-39.pyc delete mode 100644 scan_data/__pycache__/example.cpython-39.pyc delete mode 100644 scan_data/example.py delete mode 100644 scan_data/receive_queue.py create mode 100644 scheduled_task/queue_storage.py delete mode 100644 upload/__init__.py delete mode 100644 upload/__pycache__/__init__.cpython-39.pyc delete mode 100644 upload/__pycache__/upload_client.cpython-39.pyc delete mode 100644 util/__pycache__/file_store_path.cpython-39.pyc delete mode 100644 util/__pycache__/http_file_upload.cpython-39.pyc delete mode 100644 util/__pycache__/remote_sensing_util.cpython-39.pyc delete mode 100644 util/__pycache__/scan_file_util.cpython-39.pyc delete mode 100644 util/file_store_path.py delete mode 100644 util/file_util.py delete mode 100644 util/find_GPS_image.py delete mode 100644 util/http_file_upload.py delete mode 100644 util/remote_sensing_util.py delete mode 100644 util/scan_file_util.py rename {upload => util}/upload_client.py (69%) delete mode 100644 util/zxby.py diff --git a/__pycache__/app.cpython-39.pyc b/__pycache__/app.cpython-39.pyc deleted file mode 100644 index dc204d474d3d469e6762e85b9760819e39741842..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 494 zcmYjNO-sW-5Z%otX_}VME<|BJsM zc=P06=*ig%wgdBcvpc+*H*_=#5yaQ$lI~)JzHPA`1d0pDIt5^eVSyx0Ft*$gjwA_L zb0S>nB_2kcFn56ypL+`g=)Vt`_ko$u2CoR-13qNI+%fmue+=1>g>P0Xx<%vo2PU8~ z?!9Y4>p761RArOVssbM7hSP@Iswa#;t?)^1`?mx3N0*Sb3&1dFI@6c*9P25n`SzW60fjMjBV~CkP>B1n^G%dubS-TT1u@ZxmHE_Fezx>XwwHello , World' \ No newline at end of file diff --git a/app.py b/app.py deleted file mode 100644 index 87a9632..0000000 --- a/app.py +++ /dev/null @@ -1,16 +0,0 @@ -from flask import Flask - -from application import init_app -from common.config.factory import create_app - -app = init_app('dev') -app = create_app() - - -@app.route('/') -def hello_world(): - return 'Hello World!' - - -if __name__ == '__main__': - app.run() diff --git a/application/__init__.py b/application/__init__.py index 0742144..89b038f 100644 --- a/application/__init__.py +++ b/application/__init__.py @@ -1,15 +1,10 @@ # 项目初始化文件 -import logging -from logging.handlers import RotatingFileHandler from flask import Flask -from redis import StrictRedis -from flask_wtf.csrf import CSRFProtect from flask_session import Session -from flask_sqlalchemy import SQLAlchemy +from flask_wtf.csrf import CSRFProtect from application.settings.dev import DevelopmentConfig from application.settings.prop import ProductionConfig -from common.config.factory import create_app config = { 'dev': DevelopmentConfig, diff --git a/common/config/factory.py b/common/config/factory.py index 69154c3..d0d0ea2 100644 --- a/common/config/factory.py +++ b/common/config/factory.py @@ -3,14 +3,22 @@ Author : XinYi Song Time : 2021/11/23 9:47 Desc: """ -from flask import Flask +from flask import Flask, Blueprint from flask_apscheduler import APScheduler +from flask_cors import CORS scheduler = APScheduler() +def init_blueprint_v1(): + bp_v1 = Blueprint('/', __name__) + # user.api.register(bp_v1) + return bp_v1 + + def create_app(): app = Flask(__name__) + app.register_blueprint(init_blueprint_v1(), url_prefix='/') # 配置任务，不然无法启动任务 app.config.update( { @@ -80,6 +88,7 @@ def create_app(): ] } ) + CORS(app) scheduler.init_app(app) scheduler.start() return app diff --git a/scan_data/GetMetaInfo.py b/data_meta/GetMetaInfo.py similarity index 75% rename from scan_data/GetMetaInfo.py rename to data_meta/GetMetaInfo.py index 1151087..96c3732 100644 --- a/scan_data/GetMetaInfo.py +++ b/data_meta/GetMetaInfo.py @@ -1,16 +1,16 @@ +import io from xml.dom import minidom from osgeo import gdal -from osgeo import ogr -from osgeo import gdalconst import h5py from PIL import Image import numpy as np -import tarfile -import zipfile import re import os -import io import sys +import time +import datetime +import tarfile +import zipfile def exe_path(): @@ -27,6 +27,14 @@ def exe_path(): os.environ['PROJ_LIB'] = exe_path() + "/PROJ" +def GetTimestamp(date) : + # 转换成时间数组 + timeArray = time.strptime(date, "%Y-%m-%d %H:%M:%S") + # 转换成时间戳 + timestamp = time.mktime(timeArray) + + return timestamp + def uint16to8(bands, lower_percent=0.001, higher_percent=99.999): """ @@ -903,7 +911,7 @@ def GetSentinel2Data(in_file, xml_path, thumbnail_path): try: in_path, basename = os.path.split(in_file) with zipfile.ZipFile(in_file, 'r', zipfile.ZIP_DEFLATED) as zip_file: - extensions = ('_B02_60m.jp2', '_B03_60m.jp2', '_B04_60m.jp2', '.SAFE/MTD_MSIL2A.xml') + extensions = ('_B02.jp2', '_B03.jp2', '_B04.jp2', '.SAFE/MTD_MSIL1C') file_list = [file for file in zip_file.namelist() if file.endswith(extensions)] file_list.sort() @@ -957,7 +965,7 @@ def GetSentinel2Data(in_file, xml_path, thumbnail_path): del img # 解压多光谱XML文件 - if file_list[3].endswith('.SAFE/MTD_MSIL2A.xml'): + if file_list[3].endswith('.SAFE/MTD_MSIL1C'): # 生成XML文件 xmlFileName = os.path.splitext(basename)[0] + ".xml" xmlPath = os.path.join(xml_path, xmlFileName) @@ -1489,66 +1497,529 @@ def GetZY3Data(in_file, xml_path, thumbnail_path): return {"code": -1, "msg": str(e)} +def GetLandsatData(in_file, thumbnail_path, txt_path) : + try : + in_path, basename = os.path.split(in_file) + with tarfile.open(in_file, mode='r') as tar_file : + extensions = ('_B2.TIF', '_B3.TIF', '_B4.TIF', '_MTL.txt') + file_list = [file for file in tar_file.getnames() if file.endswith(extensions)] + file_list.sort() + + # 生成缩略图 + ThumbnailName = os.path.splitext(basename)[0] + "_thumb.jpg" + ThumbnailPath = os.path.join(thumbnail_path, ThumbnailName) + + rgb_list = [] + for file in file_list[:3] : + path = '/vsitar/%s/%s' % (in_file, file) + sub_dataset = gdal.Open(path) + sub_array = sub_dataset.ReadAsArray() + rgb_list.append(sub_array) + img_data = np.array([rgb_list[2], rgb_list[1], rgb_list[0]]) + img_data = uint16to8(img_data) + + # Array转Image + img_data2 = np.transpose(img_data, (1, 2, 0)) + img_data2 = img_data2[:, :, : :-1] + img = Image.fromarray(img_data2) + # 压缩图片大小 + if img_data.shape[1] > img_data.shape[2] : + width = 512 + height = int(width / img_data.shape[1] * img_data.shape[2]) + else : + height = 512 + width = int(height / img_data.shape[1] * img_data.shape[2]) + img.thumbnail((width, height)) + img.save(ThumbnailPath, "PNG") + + # 释放内存 + del rgb_list + del img_data + del img_data2 + del img + + Landsat_dict = {} + + if file_list[3].endswith('_MTL.txt') : + # 读取信息 + TxTPath = os.path.join(txt_path, file_list[3]) + tar_file.extract(file_list[3], txt_path) + + fr = open(TxTPath, 'r') + dic = {} + keys = [] # 用来存储读取的顺序 + for line in fr : + if '=' in line : + v = line.strip().split(' = ') + dic[v[0]] = v[1] + keys.append(v[0]) + fr.close() + + # 读取信息 + cloud_percent = dic['CLOUD_COVER'] + ImageGSD = '30' + ProjectedCoordinates = dic['MAP_PROJECTION'] + + # 产品日期 + FILE_DATE = dic['FILE_DATE'] + FILE_DATE = FILE_DATE.split("Z")[0].replace("T", " ") + ProduceTime = str(GetTimestamp(FILE_DATE)) + + # 经纬度 + TopLeftLatitude = dic['CORNER_UL_LAT_PRODUCT'] # 左上纬度 + TopLeftLongitude = dic['CORNER_UL_LON_PRODUCT'] # 左上经度 + TopRightLatitude = dic['CORNER_UR_LAT_PRODUCT'] # 右上纬度 + TopRightLongitude = dic['CORNER_UR_LON_PRODUCT'] # 右上经度 + BottomRightLatitude = dic['CORNER_LR_LAT_PRODUCT'] # 右下纬度 + BottomRightLongitude = dic['CORNER_LR_LON_PRODUCT'] # 右下经度 + BottomLeftLatitude = dic['CORNER_LL_LAT_PRODUCT'] # 左下纬度 + BottomLeftLongitude = dic['CORNER_LL_LON_PRODUCT'] # 左下经度 + + boundaryGeomStr = f'POLYGON(({TopLeftLongitude} {TopLeftLatitude},' \ + f'{TopRightLongitude} {TopRightLatitude},' \ + f'{BottomRightLongitude} {BottomRightLatitude},' \ + f'{BottomLeftLongitude} {BottomLeftLatitude},' \ + f'{TopLeftLongitude} {TopLeftLatitude}))' + + # 生成字典 + Landsat_dict = {"ProduceTime" : ProduceTime, + "StartTime" : "", + "StopTime" : "", + "CloudPercent" : cloud_percent, + "boundaryGeomStr" : boundaryGeomStr, + # "bands" : "1,2,3,4,5,6,7,8,9,10,11,12", + "ImageGSD" : ImageGSD, + # "ProjectedCoordinates" : ProjectedCoordinates, + # "CollectionCode" : '', + "ThumbnailName" : ThumbnailName, + "ThumbnailPath" : ThumbnailPath, + "xmlPath" : TxTPath, + "xmlFileName" : file_list[3], + "DirectoryDepth" : "month"} + + if not Landsat_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return Landsat_dict + + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetSentinel2Data(in_file, xml_path, thumbnail_path) : + """ + 获取哨兵2卫星元数据 + :param thumbnail_path: + :param in_file: + :return: 元数据字典 + """ + try : + in_path, basename = os.path.split(in_file) + with zipfile.ZipFile(in_file, 'r', zipfile.ZIP_DEFLATED) as zip_file : + extensions = ('_B02.jp2', '_B03.jp2', '_B04.jp2', 'MTD_MSIL1C.xml') + file_list = [file for file in zip_file.namelist() if file.endswith(extensions)] + file_list.sort() + + # 生成缩略图 + ThumbnailName = os.path.splitext(basename)[0] + "_thumb.jpg" + ThumbnailPath = os.path.join(thumbnail_path, ThumbnailName) + + rgb_list = [] + for file in file_list[:3] : + sub_dataset = gdal.Open('/vsizip/%s/%s' % (in_file, file)) + sub_array = sub_dataset.ReadAsArray() + rgb_list.append(sub_array) + img_data = np.array([rgb_list[2], rgb_list[1], rgb_list[0]]) + img_data = uint16to8(img_data) + + # Array转Image + img_data2 = np.transpose(img_data, (1, 2, 0)) + img_data2 = img_data2[:, :, : :-1] + img = Image.fromarray(img_data2) + # 压缩图片大小 + if img_data.shape[1] > img_data.shape[2] : + width = 512 + height = int(width / img_data.shape[1] * img_data.shape[2]) + else : + height = 512 + width = int(height / img_data.shape[1] * img_data.shape[2]) + img.thumbnail((width, height)) + img.save(ThumbnailPath, "PNG") + + # 释放内存 + del rgb_list + del img_data + del img_data2 + del img + + S2_dict = {} + + if file_list[3].endswith('MTD_MSIL1C.xml') : + # 生成xml文件 + xmlFileName = os.path.splitext(basename)[0] + ".xml" + xmlPath = os.path.join(xml_path, xmlFileName) + meta_data = zip_file.read(file_list[3]) + with open(xmlPath, "wb") as fout : + fout.write(meta_data) + + # 读取信息 + meta_content = zip_file.open(file_list[3]) + dom = minidom.parse(meta_content) + cloud_percent = dom.getElementsByTagName('n1:Quality_Indicators_Info')[ + 0].getElementsByTagName('Cloud_Coverage_Assessment')[0].firstChild.data + ImageGSD = '10, 20, 60' + + # 产品日期 + GENERATION_TIME = dom.getElementsByTagName('n1:General_Info')[0].getElementsByTagName('Product_Info')[ + 0].getElementsByTagName('GENERATION_TIME')[0].firstChild.data + GENERATION_TIME = GENERATION_TIME.split(".")[0].replace("T", " ") + ProduceTime = str(GetTimestamp(GENERATION_TIME)) + + PRODUCT_START_TIME = \ + dom.getElementsByTagName('n1:General_Info')[0].getElementsByTagName('Product_Info')[ + 0].getElementsByTagName('PRODUCT_START_TIME')[0].firstChild.data + PRODUCT_START_TIME = PRODUCT_START_TIME.split(".")[0].replace("T", " ") + StartTime = str(GetTimestamp(PRODUCT_START_TIME)) + + PRODUCT_STOP_TIME = dom.getElementsByTagName('n1:General_Info')[0].getElementsByTagName('Product_Info')[ + 0].getElementsByTagName('PRODUCT_STOP_TIME')[0].firstChild.data + PRODUCT_STOP_TIME = PRODUCT_STOP_TIME.split(".")[0].replace("T", " ") + StopTime = str(GetTimestamp(PRODUCT_STOP_TIME)) + + # 经纬度 + lon_lat = dom.getElementsByTagName('n1:Geometric_Info')[0].getElementsByTagName('Product_Footprint')[ + 0].getElementsByTagName('Product_Footprint')[0].getElementsByTagName('Global_Footprint')[ + 0].getElementsByTagName('EXT_POS_LIST')[0].firstChild.data + TopLeftLatitude = re.split(r'\s', lon_lat)[0] # 左上纬度 + TopLeftLongitude = re.split(r'\s', lon_lat)[1] # 左上经度 + TopRightLatitude = re.split(r'\s', lon_lat)[2] # 右上纬度 + TopRightLongitude = re.split(r'\s', lon_lat)[3] # 右上经度 + BottomRightLatitude = re.split(r'\s', lon_lat)[4] # 右下纬度 + BottomRightLongitude = re.split(r'\s', lon_lat)[5] # 右下经度 + BottomLeftLatitude = re.split(r'\s', lon_lat)[6] # 左下纬度 + BottomLeftLongitude = re.split(r'\s', lon_lat)[7] # 左下经度 + + boundaryGeomStr = f'POLYGON(({TopLeftLongitude} {TopLeftLatitude},' \ + f'{TopRightLongitude} {TopRightLatitude},' \ + f'{BottomRightLongitude} {BottomRightLatitude},' \ + f'{BottomLeftLongitude} {BottomLeftLatitude},' \ + f'{TopLeftLongitude} {TopLeftLatitude}))' + + # 生成字典 + S2_dict = {"ProduceTime" : ProduceTime, + "StartTime" : StartTime, + "StopTime" : StopTime, + "CloudPercent" : cloud_percent, + "boundaryGeomStr" : boundaryGeomStr, + "ImageGSD" : ImageGSD, + "ThumbnailName" : ThumbnailName, + "ThumbnailPath" : ThumbnailPath, + "xmlPath" : file_list[3], + "xmlFileName" : xmlFileName, + "DirectoryDepth" : "month"} + + if not S2_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return S2_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetModisData(in_file) : + """ + 获取MODIS卫星元数据 + :param in_file: + :return: 元数据字典 + """ + try : + datasets = gdal.Open(in_file) + # 获取hdf中的元数据 + Metadata = datasets.GetMetadata() + + # 获取信息 + PRODUCTIONDATETIME = Metadata["PRODUCTIONDATETIME"] + PRODUCTIONDATETIME = PRODUCTIONDATETIME.split(".")[0].replace("T", " ") + ProductionTime = str(GetTimestamp(PRODUCTIONDATETIME)) + + RANGEBEGINNINGDATE = Metadata["RANGEBEGINNINGDATE"] + " " + Metadata["RANGEBEGINNINGTIME"] + StartTime = str(GetTimestamp(RANGEBEGINNINGDATE)) + + RANGEENDINGDATE = Metadata["RANGEENDINGDATE"] + " " + Metadata["RANGEENDINGTIME"] + EndTime = str(GetTimestamp(RANGEENDINGDATE)) + + Latitudes = Metadata["GRINGPOINTLATITUDE.1"] # 获取四个角的维度 + LatitudesList = Latitudes.split(", ") # 采用", "进行分割 + Longitude = Metadata["GRINGPOINTLONGITUDE.1"] # 获取四个角的经度 + LongitudeList = Longitude.split(", ") # 采用", "进行分割 + TopLeftLatitude = LatitudesList[0] + TopLeftLongitude = LongitudeList[0] + TopRightLatitude = LatitudesList[1] + TopRightLongitude = LongitudeList[1] + BottomRightLatitude = LatitudesList[2] + BottomRightLongitude = LongitudeList[2] + BottomLeftLatitude = LatitudesList[3] + BottomLeftLongitude = LongitudeList[3] # 获取经纬度 + + boundaryGeomStr = f'POLYGON(({TopLeftLongitude} {TopLeftLatitude},' \ + f'{TopRightLongitude} {TopRightLatitude},' \ + f'{BottomRightLongitude} {BottomRightLatitude},' \ + f'{BottomLeftLongitude} {BottomLeftLatitude},' \ + f'{TopLeftLongitude} {TopLeftLatitude}))' + + DirectoryDepth = Metadata["DAYNIGHTFLAG"] + + modis_dict = {"ProduceTime" : ProductionTime, + "StartTime" : StartTime, + "EndTime" : EndTime, + "ImageGSD" : "", + "CloudPercent" : "0", + 'boundaryGeomStr' : boundaryGeomStr, + "ThumbnailPath" : "", + "ThumbnailName" : "", + "xmlPath" : "", + "xmlFileName" : "", + "DirectoryDepth" : DirectoryDepth + } + + if not modis_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return modis_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetGOCIData(in_file) : + """ + 获取GOCI卫星元数据 + :param in_file: + :return: 元数据字典 + """ + try : + # 读取信息 + gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES") + in_datasets = gdal.Open(in_file) + meta_data = in_datasets.GetMetadata() + + # 产品日期 + date_created = meta_data['HDFEOS_POINTS_Ephemeris_Scene_center_time'] + date_created = date_created.split(".")[0] + date_created = str(datetime.datetime.strptime(date_created, "%d-%b-%Y %H:%M:%S")) + ProductionTime = str(GetTimestamp(date_created)) + + start_time = meta_data['HDFEOS_POINTS_Ephemeris_Scene_Start_time'] + start_time = start_time.split(".")[0] + start_time = str(datetime.datetime.strptime(start_time, "%d-%b-%Y %H:%M:%S")) + StartTime = str(GetTimestamp(start_time)) + + end_time = meta_data['HDFEOS_POINTS_Ephemeris_Scene_end_time'] + end_time = end_time.split(".")[0] + end_time = str(datetime.datetime.strptime(end_time, "%d-%b-%Y %H:%M:%S")) + EndTime = str(GetTimestamp(end_time)) + + ImageGSD = meta_data['HDFEOS_POINTS_Scene_Header_pixel_spacing'].split(" ")[0] + + # 经纬度 + upper_left_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-left_latitude'] + upper_left_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-left_longitude'] + upper_right_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-right_latitude'] + upper_right_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-right_longitude'] + lower_right_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-right_latitude'] + lower_right_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-right_longitude'] + lower_left_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-left_latitude'] + lower_left_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-left_longitude'] + + boundaryGeomStr = f'POLYGON(({upper_left_longitude} {upper_left_latitude},' \ + f'{upper_right_longitude} {upper_right_latitude},' \ + f'{lower_right_longitude} {lower_right_latitude},' \ + f'{lower_left_longitude} {lower_left_latitude},' \ + f'{upper_left_longitude} {upper_left_latitude}))' + + # 构建字典 + GOCI_dict = {"ProduceTime" : ProductionTime, + "StartTime" : StartTime, + "EndTime" : EndTime, + "CloudPercent" : "0", + "boundaryGeomStr" : boundaryGeomStr, + "ImageGSD" : ImageGSD, + "ThumbnailPath" : "", + "ThumbnailName" : "", + "xmlPath" : "", + "xmlFileName" : "", + "DirectoryDepth" : "day"} + + # 判断字典是否为空 + if not GOCI_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return GOCI_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetGK2BData(in_file) : + """ + 获取GK2B_GOCI卫星元数据 + :param in_file: + :return: 元数据字典 + """ + try : + # 读取信息 + gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES") + in_datasets = gdal.Open(in_file) + meta_data = in_datasets.GetMetadata() + + # 产品日期 + produc_time = meta_data['NC_GLOBAL#slot_acquisition_time'].replace("_", "") + produc_time = produc_time[0 :4] + "-" + produc_time[4 :6] + "-" + produc_time[6 :8] + " " + \ + produc_time[8 :10] + ":" + produc_time[10 :12] + ":" + produc_time[12 :14] + ProductionTime = str(GetTimestamp(produc_time)) + + start_time = meta_data['NC_GLOBAL#observation_start_time'].replace("_", "") + start_time = start_time[0 :4] + "-" + start_time[4 :6] + "-" + start_time[6 :8] + " " + \ + start_time[8 :10] + ":" + start_time[10 :12] + ":" + start_time[12 :14] + StartTime = str(GetTimestamp(start_time)) + + end_time = meta_data['NC_GLOBAL#observation_end_time'].replace("_", "") + end_time = end_time[0 :4] + "-" + end_time[4 :6] + "-" + end_time[6 :8] + " " + \ + end_time[8 :10] + ":" + end_time[10 :12] + ":" + end_time[12 :14] + EndTime = str(GetTimestamp(end_time)) + + # 其他信息 + ImageGSD = meta_data['NC_GLOBAL#geospatial_lat_resolution'].split(" ")[0] + + # 中心经纬度 + productUpperLeftLat = meta_data['NC_GLOBAL#image_upperleft_latitude'] # 左上纬度 + productUpperLeftLong = meta_data['NC_GLOBAL#image_upperleft_longitude'] # 左上经度 + productUpperRightLat = meta_data['NC_GLOBAL#image_upperleft_latitude'] # 右上纬度 + productUpperRightLong = meta_data['NC_GLOBAL#image_lowerright_longitude'] # 右上经度 + productLowerLeftLat = meta_data['NC_GLOBAL#image_lowerright_latitude'] # 左下纬度 + productLowerLeftLong = meta_data['NC_GLOBAL#image_upperleft_longitude'] # 左下经度 + productLowerRightLat = meta_data['NC_GLOBAL#image_lowerright_latitude'] # 右下纬度 + productLowerRightLong = meta_data['NC_GLOBAL#image_lowerright_longitude'] # 右下经度 + + # 边界几何 + boundaryGeomStr = f'POLYGON(({productUpperLeftLong} {productUpperLeftLat},' \ + f'{productUpperRightLong} {productUpperRightLat},' \ + f'{productLowerRightLong} {productLowerRightLat},' \ + f'{productLowerLeftLong} {productLowerLeftLat},' \ + f'{productUpperLeftLong} {productUpperLeftLat}))' + + # 构建字典 + GK2B_dict = {"ProduceTime" : ProductionTime, + "StartTime" : StartTime, + "EndTime" : EndTime, + "CloudPercent" : "0", + "boundaryGeomStr" : boundaryGeomStr, + "ImageGSD" : ImageGSD, + "ThumbnailPath" : "", + "ThumbnailName" : "", + "xmlPath" : "", + "xmlFileName" : "", + "DirectoryDepth" : "day"} + + # 判断字典是否为空 + if not GK2B_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return GK2B_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + if __name__ == '__main__': - HJ1FilePath = r"Y:\不同传感器数据\HJ-1\HJ1A-CCD2-450-80-20090501-L20000106616.tar.gz" - JPSSFilePath = r"Y:\不同传感器数据\JPSS\VJ102IMG.A2021159.0542.002.2021159094907.nc" - ZY2FilePath = r"Y:\不同传感器数据\ZY-2\ZY02C_PMS_E115.9_N36.2_20120422_L2C0000391981.tar.gz" - ZY3FilePath = r"Y:\不同传感器数据\ZY-3\ZY3_MUX_E83.3_N43.3_20120405_L2A0000301226.tar.gz" - - S1FilePath = r'Y:\不同传感器数据\SENTINEL-1\S1A_IW_GRDH_1SDV_20210407T095634_20210407T095659_037343_046675_8E66.zip' - S2FilePath = r'Y:\不同传感器数据\SENTINEL-2\S2B_MSIL2A_20210804T024549_N0301_R132_T50SQF_20210804T053331.zip' - GF1PMSPath = r'Y:\不同传感器数据\GF-1\GF1_PMS2_E104.1_N36.6_20210308_L1A0005524847.tar.gz' - H08FilePath = r"Y:\不同传感器数据\葵花8\NC_H08_20210802_2010_R21_FLDK.06001_06001.nc" - SNPPFilePath = r"Y:\不同传感器数据\VIIRS\VNP02IMG.A2021182.0418.001.2021182100800.nc" - - GF3MDJPath = r'Y:\不同传感器数据\GF-3\GF3_MDJ_SS_024986_E120.8_N35.6_20210509_L1A_VHVV_L10005638033.tar.gz' - GF4PMIPath = r'Y:\不同传感器数据\GF-4\GF4_PMI_E119.8_N35.3_20210908_L1A0000417337.tar.gz' - S3OLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ - r'\S3B_OL_1_EFR____20210910T022645_20210910T022945_20210911T064342_0179_056_374_2340_LN1_O_NT_002.zip' - S3SLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ - r'\S3A_SL_1_RBT____20210916T020956_20210916T021256_20210917T120953_0179_076_217_2340_LN2_O_NT_004.zip' + # HJ1FilePath = r"Y:\不同传感器数据\HJ-1\HJ1A-CCD2-450-80-20090501-L20000106616.tar.gz" + # JPSSFilePath = r"Y:\不同传感器数据\JPSS\VJ102IMG.A2021159.0542.002.2021159094907.nc" + # ZY2FilePath = r"Y:\不同传感器数据\ZY-2\ZY02C_PMS_E115.9_N36.2_20120422_L2C0000391981.tar.gz" + # ZY3FilePath = r"Y:\不同传感器数据\ZY-3\ZY3_MUX_E83.3_N43.3_20120405_L2A0000301226.tar.gz" + # + # S1FilePath = r'Y:\不同传感器数据\SENTINEL-1\S1A_IW_GRDH_1SDV_20210407T095634_20210407T095659_037343_046675_8E66.zip' + # S2FilePath = r"D:\1Company\Python\RS_data_Dowload\Google_Download_New\data\S2B_MSIL1C_20210113T024049_N0209_R089_T51STB_20210113T041228.zip" + # GF1PMSPath = r'Y:\不同传感器数据\GF-1\GF1_PMS2_E104.1_N36.6_20210308_L1A0005524847.tar.gz' + # H08FilePath = r"Y:\不同传感器数据\葵花8\NC_H08_20210802_2010_R21_FLDK.06001_06001.nc" + # SNPPFilePath = r"Y:\不同传感器数据\VIIRS\VNP02IMG.A2021182.0418.001.2021182100800.nc" + # + # GF3MDJPath = r'Y:\不同传感器数据\GF-3\GF3_MDJ_SS_024986_E120.8_N35.6_20210509_L1A_VHVV_L10005638033.tar.gz' + # GF4PMIPath = r'Y:\不同传感器数据\GF-4\GF4_PMI_E119.8_N35.3_20210908_L1A0000417337.tar.gz' + # S3OLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ + # r'\S3B_OL_1_EFR____20210910T022645_20210910T022945_20210911T064342_0179_056_374_2340_LN1_O_NT_002.zip' + # S3SLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ + # r'\S3A_SL_1_RBT____20210916T020956_20210916T021256_20210917T120953_0179_076_217_2340_LN2_O_NT_004.zip' # 读取 HJ-1 元数据 - hj1_dic = GetHJ1Data(HJ1FilePath) - print(hj1_dic) + # hj1_dic = GetHJ1Data(HJ1FilePath) + # print(hj1_dic) # 读取 JPSS 元数据 - jpss_dic = GetJPSSData(JPSSFilePath) - print(jpss_dic) + # jpss_dic = GetJPSSData(JPSSFilePath) + # print(jpss_dic) # 读取 ZY2 元数据 - zy2_mux_dic, zy2_pan_dic = GetZY02CData(ZY2FilePath) - print(zy2_mux_dic) - print(zy2_pan_dic) + # zy2_mux_dic, zy2_pan_dic = GetZY02CData(ZY2FilePath) + # print(zy2_mux_dic) + # print(zy2_pan_dic) # 读取 ZY3 元数据 - zy3_dic = GetZY3Data(ZY3FilePath) - print(zy3_dic) + # zy3_dic = GetZY3Data(ZY3FilePath) + # print(zy3_dic) # 读取GF-PMS元数据 - pms_mss_dic, pms_pan_dic = GetGFPMSData(GF1PMSPath) - print(pms_mss_dic) - print(pms_pan_dic) + # pms_mss_dic, pms_pan_dic = GetGFPMSData(GF1PMSPath) + # print(pms_mss_dic) + # print(pms_pan_dic) # 读取葵花8元数据 - h8_dic = GetH08Data(H08FilePath) - print(h8_dic) + # h8_dic = GetH08Data(H08FilePath) + # print(h8_dic) # 读取 S2 元数据 - s2_dic = GetSentinel2Data(S2FilePath) - print(s2_dic) + # xml_path = r"D:\1Company\Python\RS_data_Dowload\Google_Download_New\data" + # thumbnail_path = r"D:\1Company\Python\RS_data_Dowload\Google_Download_New\data" + # s2_dic = GetSentinel2Data(S2FilePath, xml_path, thumbnail_path) + # print(s2_dic) # 读取 S1 元数据 - s1_dic = GetSentinel1Data(S1FilePath) - print(s1_dic) + # s1_dic = GetSentinel1Data(S1FilePath) + # print(s1_dic) # 读取 SNPP 元数据 - snpp_dic = GetSNPPData(SNPPFilePath) - print(snpp_dic) + # snpp_dic = GetSNPPData(SNPPFilePath) + # print(snpp_dic) # 读取 GF3 元数据 - gf3_dic = GetGF3MDJData(GF3MDJPath) - print(gf3_dic) + # gf3_dic = GetGF3MDJData(GF3MDJPath) + # print(gf3_dic) # 读取 GF4 元数据 - gf4_pms_dic, gf4_irs_dic = GetGF4PMIData(GF4PMIPath) - print(gf4_pms_dic) - print(gf4_irs_dic) + # gf4_pms_dic, gf4_irs_dic = GetGF4PMIData(GF4PMIPath) + # print(gf4_pms_dic) + # print(gf4_irs_dic) # 读取 S3 OL元数据 - s3ol_dic = GetSentinel3OLData(S3OLFilePath) - print(s3ol_dic) + # s3ol_dic = GetSentinel3OLData(S3OLFilePath) + # print(s3ol_dic) # # 读取 S3 SL元数据 # s3sl_dic = GetSentinel3SLData(S3SLFilePath) # print(s3sl_dic) + + # 读取 S2 元数据 + S2FilePath = r"F:\test\Sentinel\S2A_MSIL1C_20220102T031131_N0301_R075_T50SLG_20220102T050158.SAFE.zip" + xml_path = r"F:\test\Sentinel" + thumbnail_path = r"F:\test\Sentinel" + s2_dic = GetSentinel2Data(S2FilePath, xml_path, thumbnail_path) + print(s2_dic) + + # 读取Landsat 8 数据 + LandsatFilePath = r"F:\test\USGS_data\landsat_8_c1\LC81220342021355LGN00.tar.gz" + thumbnail_path1 = r"F:\test\USGS_data\landsat_8_c1" + txt_path = thumbnail_path1 + Landsat_dic = GetLandsatData(LandsatFilePath, thumbnail_path1, txt_path) + print(Landsat_dic) + + # 读取MODIS数据 + + MODIS_path = r"F:\test\MODIS\MOD11A1\MOD11A1.A2022038.h26v05.006.2022039112456.hdf" # MOD11A1数据信息读取 + MODIS = GetModisData(MODIS_path) + print(MODIS) + + # 读取GOCI数据 + GOCI_FilePath = r"F:\test\GOCI_Data\GOCI_L2\2020-01-01\COMS_GOCI_L2A_GA_20200101001642.CDOM.he5" + GOCI = GetGOCIData(GOCI_FilePath) + print(GOCI) + + # 读取GK2B_GOCI数据 + GK2B_FilePath = r"F:\test\GOCI_Data\GK2_GC2_L2\AC\GK2B_GOCI2_L2_20220208_011530_LA_S010_AC.nc" + GK2B = GetGK2BData(GK2B_FilePath) + print(GK2B) \ No newline at end of file diff --git a/scan_data/__init__.py b/data_meta/__init__.py similarity index 100% rename from scan_data/__init__.py rename to data_meta/__init__.py diff --git a/dms_client.db b/dms_client.db index cd1ede49e893376e1dc1ee4d94f3e0a8d89c2fcf..ac8ad38348619deeb7ffb93115f6ae58fd5ea9e3 100644 GIT binary patch delta 227 zcmZozz|?SnX@az%AOiyf7ZAgM_(UCJc|is}>yNx#>lirMS{b<;_+oh-I482t=U>Ox z%9F~mXk#Ng8-J5H7rVH%HDj}DNn%n?>g0!FAHnSA`;RgUEu|6LF delta 153 zcmZoTz|^pSX@az%00RR9ClJGc=tLc3Spfz;>yNx#>lirL=QD6O@Wt{va86{O&%cgi zAy4XNK>=m<&GXrr7?m1xx!A?!$(jmQBv*Qr+Cg kSr*2{%s-!je;xmP{%1f#Ci3&~GBL|D!WC~`n;)zI0NI@>oB#j- diff --git a/requirements.txt b/requirements.txt index 32c5b47..49ca4fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,22 @@ affine==2.3.0 attrs==21.4.0 certifi==2021.10.8 -click==8.0.3 +click==8.0.4 click-plugins==1.1.1 cligj==0.7.2 colorama==0.4.4 -GDAL @ file:///F:/Soft/Python%E5%BC%80%E5%8F%91/windows%E9%80%82%E9%85%8D%E5%BA%93/GDAL-3.4.1-cp310-cp310-win_amd64.whl numpy==1.22.2 pyparsing==3.0.7 pyproj==3.3.0 -rasterio @ file:///F:/Soft/Python%E5%BC%80%E5%8F%91/windows%E9%80%82%E9%85%8D%E5%BA%93/rasterio-1.2.10-cp310-cp310-win_amd64.whl six==1.16.0 snuggs==1.4.7 +flask-cors~=3.0.10 + +APScheduler~=3.8.1 +Flask~=2.0.3 +Pillow~=9.0.1 +redis~=4.1.4 +h5py~=3.6.0 +requests~=2.27.1 +dict2xml~=1.7.1 +xmltodict~=0.12.0 \ No newline at end of file diff --git a/run.py b/run.py new file mode 100644 index 0000000..536819a --- /dev/null +++ b/run.py @@ -0,0 +1,8 @@ + +from common.config.factory import create_app + +app = create_app() + +if __name__ == '__main__': + print(app.url_map) + app.run(host="192.168.2.105", port=9550, debug=True, use_reloader=False) diff --git a/scan_data/__pycache__/GetMetaInfo.cpython-39.pyc b/scan_data/__pycache__/GetMetaInfo.cpython-39.pyc deleted file mode 100644 index 5e6cf275fbae90f5f50cf9974a84e84ad1d9cf8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30380 zcmeHw3wRsXbtW(X$N`_=`zaEns0VG5#)G6NMRlqNCCi~GMxxkj$Vu}-d&>v61?8{|eD8{{Ur8OKI>gWQ5+ zliVu%acq{`Vd%^GJd`)aC*X@(~!Rs^Y_Vckm)`_~? zz~9FC=GYeG2aq4&d{b;Q^4pQ$&iTgJEy(Xceh24kW49vz4&>j#`MOvapKGV|8{q9^ zULf`xeB2Q0M%~-Nzn${3XVlR-aE0bkr&BMSi6`Q6at2BFq#T{b(UqK3aPU%gJef$T z!^n9K%tR++(=9cGSr1#01vlX#uN z9JMR*lUGkKm{5}Q zu|%h+7oLcwqN=LsuGHC-E+kXBdp4?`&&XV0o9l8Li((Zo#js63N8ni`KLB63uX9^D^P z560B!fy6|zXZEZv;v5m`m%2(7fH(?@gfgd6iPXI3wvoe!?vzBLk)Z>-U;oEvfAUuc zkB_|b^|J2gKl!6y-t(K|^sBc16(b?y%lG&%kWd{{PK~G_J0BC$F3mCJl7$9G+O4`Z z_j%`D$9*13Mlnz1#7kx>m6F5 zR#5M#c4%G=xh6-QBVB~rMfen)bnK*ZN*A7VBVDAq^3T81ajIQ&VGelap2sEe177)^ zM=!nd;_{QPEuX*mqgOtG#Osf~KiGfi)yFQs`iBEJaIGD)Q6)O#KORlUsU7~yuYGv= z!e>?%FRU!SvhvX{tvvRez0!2@bWDlN#+30`LfzqCdFp+aUVZ%akH5D3$ipjN{D{5g ziTLD+ysB3||FOKP3QtF4Hq!4wR?nhuC&qu}mmtHP1-g)!m5SzLN-`Chj-QN4p1F8J z9nd|g6Vch2?opzN$(U4RjtDcWvsm}=rAVbYgO$p1M}%MGoT@t$x-+gjqq_6B?i|;h zvhGx~(^aX!AapOCG1#vr2U?L$)q^-hpR>qW;&kKFf=>_9CC+-MvJqwDkQF5TF`_=%(f*bW}C3+(p*b;i)aqav_RTTWttmh9?gR?0l#{haqb4kd9>h~ zF7WXla5_%;)dI~a3$jagKjymFkuK1LDa<9=Bh%L<7d{5Lbhkqm-43_oJY-kG7j&dP z5O6s8b1$e?uqZC!D2tds-np|FCO^vMN1j}M`K2?{Gb>NL|I*7}S^nBrmLL8&X2XFPt}~&84p|tvvWednI(l?}Q!xOqWKneHuvRBT7=98|R<@ zUUaEDPha)SsT17;m?2^;F`ks;iOH)@ukI}Bbn6B2gp9c;xf8L|(~>tTqm&rRW?aQ^ zr&pn;A&GmF<8w1quXG%*C^a@X9ph|qbapnDkoTO3Ps>u7Q8pGkqYfwK7~Xhz?YWbv zAc?1yxEeb%Gp&2Zr<18zXQ3`y7cGg_mFc2vo=0~p$)u{gPR7nk-UK>}@VnAEv1)c1 z=e+0OP#*@Gss?e0qEO^65CKi3L1rwt=YK$NGRz&v z669&xh4+_h%B=yz$iid7g8Sm*c#F9nb-dp(;lz8(^N@)7 z<2TJ7!7+Oj%pSo(k9apy?R3#~dMxK2eM?T@k@J%&D_+YM4`R;h`uS$&?c4Qy zy}%>c=j*@L^MxGByde^mLABCQoq|GSzW<(Ke&=``Lic%!sx{t3Kk zMj^}8ss+#4E~g{i=5)+&R;$$-t?j&%(YV)08?6c)_u{PSwsUQx<}u(1AO%@>%iN=Q zd$JHR;4&Zzpp1Xy;OO#WpI`adr!vyaF2~dpb2G;i(fD)(fYbmlCWeB1V3eA}j3*4K zmm}-4a?&PDb*J(ymJBlx7oF~_Wd}z`LlHh%&(!SX)v}RY!}h#L<#P!zZ3{Gf2kJ!*h+oEqX)PMwacC!Q5_p(mib z39Ew-h_A*HsW=uchWuh+RJ3zCo>Iq?GqbnfkM4hluVda1fDSYBpz{Rq?8K!rZdGaRpr@m9u+ev+9W}GzFFIAV2Q_S$TeG<*7$k zUS7KVl`pS+atVm?^-nxxfJNg+fXCfgtB`s|p*0#n{_J#I)eG@3X(B4O5>uRvsf49c z$~!1qh^eVW$5p-9Oi)druUJ%O=%;(Iyh+7$A)eGl+y`LC1BdhiImVN!8l9O?Ftz*a zSagyvqi77$`f!{OWDU=lXvxlaLa)=1cBjw#h4igzAGDhSsB1eeaP2?UT$3Udw$%)8J zDix9A<0@xnX+4jO?u;2Pj(1XBetVSIUleqL6m)68zHX<_S?P2;Yh7hXVVxy3+EPBC zVV_gP(qBZ4O4O|MFg4PYH$G@1{L)QdLHGrKUBdr!)7LHhBYnj}!D;)rY-y+P6S1Pv zWu6N?WzOZQr1SjB&$UAkc?DU7Yb$Xo7jPePu)ZIR4A$R?ci@6+p0yV1VtwJ_SKb8H zw~+Z8#QH$y&)d*`E;7$U`zA667s#zxD`sR~l0oJrK<4EpGB1^V=`vvVa>DAg4qe@ODUK?eW%hQ>atovVR;>nj-mekbZ{<~h zUn7LysuA+{YlQq;c}3v2)o%ml-=LKf>c@Iv12BK3R;86O<_9h$jWRHQ%S~ZIA@#i+ zOxUs@q<#SuxG94ITTBF4x!^k64+Pi>1c-I0TBWs~cjlnJ)^n|l`f$vkzRhz_;`g)) z^@W!&e)ZtqJAd@bBm4Jthy9#<^oC#{z1edhpCtxzm^Y3Cl`zJq^n!E>SBrYG5Rg?4 zZyM++JE;4eMEG^g>REaxE4x5`L5bZU^KJZw4&z3whvf_q_As^BOJtD9J|g=;hLt^( zd_NJ!XodUtg(K)rDw{QQJo_PpfY+qrI27TK%nZ~#DFSc03s_XD7*0Z$9-TfMOC_i0 zNc2i2CVDvtkTN~*3U-C0Le;=-x;Sm2H}ABCo+J^a49BFRCzGog$(TNfA+r%n^Bj$Y~;sN&E^a@sxQGy(|kwXrd~=MI{=MG?8;e z)*ADJRPftG&J*Eldl#j$NHcro3>e-`rFmE(ix9G?;TY;k;^ah_v4QdsHRDN`MF*uh zbML39k@`ar8$196u;2k5v!K8Y0MP7gc2x?15fIhFFGZwoklrk8utNx^@H5f3!&Lwg zUV!l^k7K+!Vw^y5MvR~HK#Y6l`>&(2({EUeKMVyIz4~D`#M22Qz_S`NulSHNquK=tL6GMsqlOZ3M?+E@O(0fSc^j{JfBQZ zz~Yb!&nJ_Rw>YH2^T{N>Ee@&hd@?EQEDovgd@?EQEDou7e6ruhfiMQA&BkeAPP>iM zDsN;#-oe6t6ASnb7Vw)iQtfrH;NPURa=Ha{ldb+{h=k8X%gJ(y1o>@@*e8-$y;N?AcaLnh9ROY!O{gyrVTYmlAF&_vN zuJ_!FV6x0U_x5W&cc}j|?-y48c~S*AP|vei#p=Jd1&`99wJ&&50nDOXjkgoje{EWO zM*UX}_1_tQzm2T^t5It;xD~MaugFLntyuk6n%;PBBY+?u83>j_{ns`3bG(sP0l{7X z!I6Up3?SGGWgwDk0zqiGP-*$%7cc+8gUc5l{r>wNy!`bKzW&nF%b$M2-*@T7&#ip< z>6PbR@b?8~&;d#h9H#o0Ui<Xnfj28DGYYY{aWMId-8^Dfw5yNNJ;=WNFN!l(0OrD_>Bg&2}qEsFu z?iw)CrmGqO(xm5QV;PF692VEY<>l9+E6#%FJRq8fK^d3k0nrf};96)ji$Sxv^cX5j zC6g1qk=Yqjb;UX80I!Ent;sur`Ijo8N$Ab0ubMOXw3(Gwq{#9c>Z=@;67Z@(n01v} z2?+P5R9N{~wES0ozJeSj*{bvHaESsrLYQi-58-~~;NQ())Zj}~J0-e+e}(x!S%rVe zgzX~t2ie&GmpjON57&WP1Y92Agrj<<;6?-22KQsa#UYrY3uO;#qIMb>gfK9ty+{`# zEz05ra^@+Ly^C(bBoD!XhD_e1a9ZUU@q}F-!E98b7A}h9aJK~WGSXf$uCoc5F(wo% z#lV{7TvAS^H#+x{bFlVk`$kJ8m@#bTuf6bnRJH3dgNU?)1(G&oMW7qA!sf~{4nf!55@rc8M? zng8n&e$=Q#jkmBK4lfOaUkZzG%(yci9acooT zb8BMk?NCgSs>;tu)~gX>^)r(7YJN@=lJ#osoNnWEwmzw_HfoJ>JAJf`Y|5lmf#%T~ zw0cv8)uDAj4fUE_#b#`Q-5WYgHBCPpHSYP|-hMMff3MjD21v|uD6rZ=72 z1SG;oMk3^Nvt^F7BsL^+`GYT9{=gp%{L`qP=tah2H>-#;#Y%+KHEbp{QibGd6^YBk ztTG8}Be3zF5@gtMJDu=^G(?P`{05zP0})mSFsVRR z_wJmGC1Pi0mD>dxJoAtd#h{d;!}MfM#S8XVqr za4>RUG;-J2z5(5J@8IZ_LY(AE0THaq9nZQ9%@xBBz8QnyW_efO>Eh5`Bg6X{WauTZ z5yc{gBaf_C&B3KeiA+Pw6v^nLs_jfmHFf2W>5SDN@o&Hms$?<6Q92)KpkShjo>d~0 zCG`>O)jFH%RXfXEWP>UJcnLv$M#5Z#b*h#>q+~0B za;-?Hg@vvKu%+2q<}9&nXD+mM7P&;XNa{G$Fh139637(zER`B;V_(Neb3g?t)iR($ z#}5;r;zMr)RKW8Mkb=cL`M!~u2ecs1NP-n?m-4Dq;w52j3UmSiMJCq-ArSt=EjGC( z95J~iDwI-&zYL4{QWf545Mia@!?p$AI{`MXvaEoIhcN(*>@~%FnF{re27fyEsg+Ju z$VCwIbSyT-ydRLor&dBeW0pe9o8>I#DOz8P^1>AN)q?*gC(1t z7u-7L0=i_lxW05fY68Zj8*r?}vC+&o;aG=!GmiB*ZZOMR%zUd^?w1=_xmCqVtVV_k zRSXpx86LoD1W3@xu%U`!L!+&J6W~D`tGH_AW+)))wK};ODS`)8S|#nJfV!(4$_gR% z2&`1tQ$fNW9{P0}_C4SZw3{%YenB|<*MJFz!h){VQdoFRg$1`XVS=Hs5J_>7feCGM z-^OgW!-NMvvb^xf%I`mT`PF9(tnm7mUw-}NPeILa=g8=2caSs}mPamJz=sd*+SMHj zY`qgwLiLX%r=yC06vAs7eikwR=-HGSoAIY+qvNr0B|4#Y+LaM&8nrSn_&4MCWSEg? zov-{Jk&h9fHT)GXrJe#AHf&3KD9>W~la%^>A`2jTnc@CJ79uhtWo;C?IE42jtv%Vm zqkG`q0|$}1;iRIT*iAk%usdb_VD#F%V<~kl{$O-Xe%&bi$k*t|5axO2q)*d$^MrXr zo-kKFLzU=_%%WR)mQtT1@_8a;1yR00ghaCP91#-8$`^@{C|15igoLp2M?{_{@?|0+ zBD9)UcwK)Qsm^Px>$B4LkEzWIM7~1gt00n?h$}2JB`=f>MpAi+ieD!33XxZdyhdb# z2rudX6{Ye_NThzzt4yQdT?C)VMh90cy?PZBPcxUxCOoQJW3Ov)sj}7Y5ba&a?>Eb| z_siT>{o1&wY)D)0)U4_6Vl!i91uX$A`XLgiIs)00q8R{53pD^_C1mLvkgkxdq6_la zg`@p*=iq(dABIms5|U$KxtwhJOTJ3W=DK zIuW0RxE&oH8Le5w;=KOsR z#oQX}L0Si*AH!|IiN9XcTlCH_{P{XWOo5!Pi zbr~L3eP+G-JRa3+$ndZ#H0w3y@u*&tT*tnR^;$L7c@6B_SkFF;4ZI?(XY+OguL$dT z<=0@V--z{Jvw4r2=pHFb=2>sRv-ajZYdCP^J!{(KBBY+Nn-xwe^fA3Ao3&i+ zWYu&V*68Fd$t}z^r{OIrrY&ztQcZWvk*bTunIHd*eKN}*erEZ?3&FM2(^Sda4zrfr zvlZvcK3hSK{A0ekLv+nuGgKj~s5aJ;3m7tMu;<_LozVX5-eTolXWv$Pjonc-Gaxyhrq z=@2>Y7uy(8adbXDgH<7|L}^QE>JDg9vHtY3X_mQS3SNnnn!k0e0nRG12ja2Qx6MRn zyOYqHPDju7oWy{N$Fn;&t_nv}^TN@ttX|r%duO%C85uo)%e78&YA!y0vU@t2JlQjg zbq%SV=fk^ZXzv9!!bFbT(RF~f3#4!lC}S%*VlO$O3&8+;Oxm}k(54IHN{#&l$LPtd zg$A;#%{(+PNG-CEz_vkz9Vqp}`S>hv`Y4p*v#j#|47Kx}adUIG*14 z3B#xIRx0-Ju3nperAjU5h_FNDT840&Yh|7xTo*AC-ezH2?q)7aFPMqV9FHlyj>gU^ zs10o1m06?qf2OnlCmLNLd{`r}Al#fKyNG{v#>%{KG`Kz@@DU;)2i!z#I!F_F3xsKk zkc;JIN$Ons;v>*@_pP8Vku+Ejtf7?!6c;xlmlN+^-5b3fNr2p1KU z@EN<%sco@pPdmZ?>E?1;up0iBShRi*W6uE#r0mFeT;A^nSa8p;0Tvc{lQcF?zkaE{cs$Ec=!&DeUwaW_Q zvPGzHYMGty>b~8#x<-k=2BJ%b9qK# zTn2@44J(Y-szD0l+8l*(9TczStT6T(3S)>g5g?{srfu6UMbEo;6&VGx7hsGb1w>ne z(F*sdT5Ti?I76x$SfQK9&dp-B(DGJgZ!%}z#sSY_W2TQZ$f^vyqOj4TUgP)mF@6bG_u2Q)jB}8Xd5}bnbVs% zO?G>uekTCf7OhEZ#`cnWZHs&htn!WWt-QsA6!2YgH^9658^H;*AS#21}g`mO9i;8UhM2SSmHrMhgZ@h3U?7ov`EckrniX084>< zJH7#wLTdq~Yk;I*&30l2MV;>XlF05rDAFI8!8YO$=fXL;UMqLC+JHv~$M#~M;enyh zuD943@OL0_3}b$WO&Enp7_GH+IBO>uW~g(u^tK05aR}YpHl^_2FbvSTkQE<>#UN`Z z_&yadF#Ic|<{J~i9q%@fUE~0)0OL{YTTLN3k(3&EYj9)_L9gg!EV3(=iltH%Fl4?c z$o>Rf{{Fyx1-hW^wJ~ff9XDDwWlaf@!?B5&f`BElh+$ZfiuMm4ij3{rJv2BvU(Riz zKZ;HpEoom1#XT`{<@*PR2M_NWJ8)nJAK^`^l5hWX@;J_t<&+HGGZq;+G>TYBqhrcn<8EAmXmlt~68Qv?j}uu7NH$`> zO;@<9x2vzKziV69K-cyz+>l_fD;Ub|m{Gn%&9jhgbH0h=yHx&PK=cynz({28;J%?< zV}pAYomlP^Of&gb5H^62dQLAeBDTB%^T#{rtVtpS{*?bp1ey?sGD{>6`uz>@*1~_c ztnaSyx76kwku2&HmC570w^z9x1$uF6?l@yJxLD|3${0QqH=rhB`^(=^@4rptJP4xM z&>nUxAd}(H$_lnTJo(?L#k+|dCNfH7jL2O?ju5$<$UQ`kfk=gjJwQ8mZDx&kQeg#% zZikNiYke<%$f;T<%6R{7H7x(Nuu=d#Rv8YFw7<0(oNIXx_(&6lPl@vu*fMNDlOd&h zHSh!gH2^2=^939_OfLe!l7!`B0*e~&wp15CGq;lmgA z&5=?;v7@((VNp)>o*ZB*T>S2}fGM{wTmz7b*94^AML0(00a9{lBtQ!N6$4V0wNmQ= zQg|XTASFk|8v&%`sAvJwDie@aWdLax1JY^+q+H5?l*@AfX*C1VwQ3L`B|w|?U?5+E zwE$@y15!cxao)Yl0#eZiNb56TvRZA(0i=!V0#e4HPHkEQ_yZ`dm#fna z_%>?wiyL_B5rEoM%T#N+i8&@5Wll45Ojybs03)=r;3>6gVU7t;xlZe9|(#*8gR4A&avYPnHv0${7nz%^*Q0M|CJ0oU@kg7_J( zZH8{^7QnR@xz&VgXv=V|4Z5uabX!{(uC=QhHN<-~;Torn7A9QVa&8O5H5@Z=%|AzW zaht`*fNNp@p`ksr|6}EauU`51!j{;}(B*vN3w~8}h=UnxjnULpZqKWtfd>!6mh*t?@VL-gi zHi#EKG<2gNPq~Cky$-bdA5#D-j!R0-KpqAY{%L8JjR*%6L>_yE|3tBFp*v&eMI_+Y$Gy2 zWIG7p6bQ8lQi~w92vUn6wFpv+Ahifmiy*ZKQHxM;zA&^c&=uO=%R0Dx4A!wO22)n( z?h*7_7j!9?sg$fPH|KD2Q;2IVu(kE3z?O1_Mo2hI`2mq161gd4b(~sc;FV#k93fsF zp1MIO1r&9SJrMVo84761yLW<-oYA7?(;;9~PQGgabW(y}Nnh^KTYJrWwwibw# zN|FCeMH@x^%Z)9F#*eI<23mpvdzqhG(uK4i(^C@A?usPG2)Am70%)qQANlK zXK{?EB4mxTI8}TuvcXv#BgP1nV;PPSS%fTZRvjar2*o+EI7So^(|3}`Y{U*Bg_c#v zh!{d!nJtbHCxq~X#krP!jADOSHZh6@XxYjrR)}FwGwfqMr|`kn4nzWhyl;kQB%%N! zMX~=V4u~Q3H$dvQbIm@&b2`5^O2=Ka-HpSAHm$UN&VL!|Kjq~=Y!bx33-0Uorc%sjl_R@ z>vm<%d&lm$>)`I5J4f~#J0AC7$n)+V#0fRyote&V5Ti!KT-|2|(a1k^a>6bcNI&@jr}sLR0`x`FKazq8B=$Hp<(Y~FtSEIH@R}cQD08KMic$)Z6p_4z|5{?G zHD3&+so`%C(TMyvB5RHQK`O|~lOIv`$3%WYF*T0$keD$lOekD;<8o|w9Z zZHFEF?_{fvLL%h@XhI^T5{q1j1m2}bp|!}-zsyBTZ8Mk_Ma>Xa?L@q;Yhh7A#!h1w zDwZ($;pRF~u-R#Tj?v@@lTBb`gvpOFV%TY(Q61-DWjT+{HfaiF>ztxM$*`Fqw5QHL;sUGdt@yv9o?N+ZLMGw$RMBg(kKwz_B7ze*^FFqtH~@Yr3dZULoXYIGwq2%Y(tq%CF8PQASDwqY|%JGO&K(- zB^}lwp~%eK z8P>lWIV+B)4RbIAA>d&Pp1|=G@BnNqj=BJr@?N@t_Yuj0t2}s;UF065di0=_|4HP3 z5&7RlZvKHBqh=2g@nuK_QaS|ymF^9M_K+s%FA1d-nFa)k>{Xe;DFr4@aU&Ey;T3l- zUZKbd#wUMydtKC$QQ@zNS;SfcAH8kotLs3>e_Q;7tGG)!D5)Kc3`+VtW6Aw7LP%I< z107}1kZ^It2>nlgsKmlcntRbeN5Xv_jFk*@bPRlo++;^bg+Mdl0U;FvLjfD5W>57l zI1_))d0KqGkMl2@ZS6>i+TzcYz^}y(Bm}&}POyZ8Xp5tPglMU4At7>e%|$}x%&{XO zY#Nt?Wpsi8*#8gct{?V@E<8m}5smEzGeap;jOvT8hIk0NTuWsEOTH;UU3%!adE5 zhng5CHQVYF67n0ar-<}G(Yt_(fP{FNj_6&6>#6HhFXUrej(ikSN{)PVXXK-hdV*X} z5$aQ*MY`rPm+IBnlA%K?wq>MZn<*9BAr+A}TCh}XGXu2qk)TPlVPsffcr z2dQZ6a;F&W#!`I&Kht?HB5TK#>r28n|AJjt;@yvNu*6#}=#>9MWm&E19&tCR=Q>IvLJT2mK5`eawtZYW$Kv77}kb-#VGYHRNL z`u}2NI{pIJOcbuvkvY0g_wG1K@%0ye_tGniD-V5Y`Qy(Rk=Kusz6&I{t9#F$y`k>j zzCiat0DFW3+XH=pVD}Kk&Ikkp{r$oI9t39XnVg?HaIOAmyW`O#cLoEY0|)o_>F^obh<)k}VAk~p2Qc-5#?%|YlL!n10@U5{YjsCapmEt8u{bh18o>?SKG2Wr4+VM# zaQ%Hf=Jodlw$t^)H~z>GBB-G#Amy_R~%?HN<{k`Gd zP$UrChWXjo9|>>6ybbpTB16N$$f3yaScHDgG)-@OD7zw~G?c@;$81CC$57~ZVGV`y z!B958Z49@3dmnxtJd|yy7~EzJC1eeymxj_QC{?&GN;OF5J0S~<$y1c_5~&2ipPxuP z5Lci*-$<+wkqbl~C-Na8UnlY{BHt$RJtBWku+72%j+ZM4##$3Zng{Ih-iYeZn`R5{xj8&!J1P8B? z3R3KZh9m2`CDGdBC>5H~qv@|qWCHgY88dDlDu;wYbnJFo zbEfD?k-Vl4xvJ%CXmp7sK37q#xIuicqfNY@>4Z3guq^i=EsE3PQ5-jmQ8u-bxfOwN d%#V>0+_Xg!VRUGHXQqG;ES~Mi^|%D7{|_wA-46f& diff --git a/scan_data/__pycache__/__init__.cpython-39.pyc b/scan_data/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 0af127ebbedc22ffdec62a4930b20c9d84ee8a3a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 209 zcmYe~<>g`kg5!466PL2?cSH|_&afu%ZEx1h?N8gmWiOQ?N}*}V{)m|tcd83VGpN*3M!l z2c$fM4GXYxDrd}?W2>Kk^q;?I&m`C% zrEnLP6iLEU`{}jE4?e5jd{kY!{^Y?2K&)PS>*(>v4_DS6{=T988&wN zlWUKDUS0lmt-M?-KdAlU54DxwH!4}qf=MRxCdrsK?N8Qj{rK_2w^!ePRK5CU?UQ#J zHD|Lkv)feHKYnkUDlut+*!}*z6w(z;i(ngp|Le~HjHN=JXy^6FqDdTg(#p=8x>Cs6 zw9ixAStD%Q;*a=%35YB>M8zG^!OHM^~6?>Jjp3g57{2dddTq5l!r1N!XBa@ z38W2v{OPfw{)uU@II-Z6`H6gy&N_CnVAwfh0_WTbH)Gh7*r3LQdC|z_EweXY^fd0> z(c{$V;{oP&0T9$?6hSd0!@CFGUZ7*B3y}_xWkjw~f5Lz_5SbMO zktvoXCRN}zF#)3TNJ-{0CWB023dqE=OnG$5-NF*g@HHjiE1Y^GNka@_5ldKJk**&s zg_t-8afcPmKM99d;69!guqq3(umtziZaylwKRF^G|GfZ^h01CLFjgVfv_cWw^pCH4 z^=9?{pD$Xu+PiN(zJIU!r+d{m--k%5{_dC6<F)5e3dSqtll zKz7c&NXHy(dXh_tCp(~^s|zGc&5Jq9Q!!?B^9s+So0;CZv*U9P&oqJNS&*I<>%aJog3Q1jnQa1201RuVlxoyr8teV z_!7s_a)Lp10ACU;fmN(R{@sLFTb6hpE+uGFxtS$kW)q9gwcrTkW{#uX!n}L~ktB|l zn&}SKydmASDQ|;YQ);ofMhj@Pe2GRgYbv+G{FZ6azhO<~Hn>G`zAu-;m*STamzuWi zW&(C|M}0RD+W{+RrR_{*txGV6skjOJZUfv582Gpaa0l%GY6ri`aQ|E3Tr}Wz`rNh# zF08NA1)A-Ey8(9q-dP{-0^AAX9>86Ich}{6>f^n2c@lTCoi_#C%DMsWWIaB;%cqk* zz1OGr`1Ed{ey+dkc6(_bYh&%K1NX3ftc!Kx9-u-@VXbTj-i`O%N|g4q{WnDSDpd8o zIC)DvFQf(9$@VYtY_^||WPx_w5R3mnLg@e!7GI)#tp}D6*T7mp2Qx#$1;{|91D6h@ z>-dOpB*;no3V(x0_mjoi+kdQHe|PQjJJlOkYrnd@_V72=m5*!h-C6zg{_6cZPaa%- zbv&IueDukKYnzz}%H-IYfq}!vjvRW`fn1&(cPxV>(~t%%3vP8XU396LOS*X@V`hjk zO;cOaR?eEt8+0}(Y4yrVXA8M0+sM8^l(mB_wPYz2m&;(iicEjTiDe(@>dTU}#YrKQnq^Ay7sXV{6FdnU z#Z*F%22PvYJMvY&wyy#3VuO}bz~e!?;x$ci=L}LDF`ZnR62conPxX7ok@S$>21e%0 z3^nm{?_Do3=vWqTVLt|WqK}{3IL>i6=!PNR4=)T`=<}mMoP+}Aa}z^Z0wx?%95VB8 zf;=DB?pA8IuGQ{W*6w{;yIp~rx%$DCpd1Fr-rL)o5JCo}6M91}H9$(P^ z~Uz;geI1fKK`5E z<{-tvW_Xcra6gn+c;l?M=7Uz`HRut@s5wnX4XU?&feH6!EB7q_39j4Ts2gy(@`l|} z-g$4g`<&Tm*XMr0ZUYyT$uaJ`HkWtZNu13@ZV;jW2gs3#17u)iZaj%6q7{czY@$q5lKbxN-!g!_$YX#1QMUg{1e6h z`JW{ISLUBAe$GENT(_jvabP!q{|WaB2fU55T|20zn8u6XEA8; z6vv%0o#3`eq7AF?re!~g&Q diff --git a/scan_data/example.py b/scan_data/example.py deleted file mode 100644 index 101bb41..0000000 --- a/scan_data/example.py +++ /dev/null @@ -1,165 +0,0 @@ -from xml.dom import minidom -from osgeo import gdal -from PIL import Image -import numpy as np -import os - - -def uint16to8(bands, lower_percent=0.001, higher_percent=99.999): - """ - 拉伸图像：图片16位转8位 - :param bands: 输入栅格数据 - :param lower_percent: 最低百分比 - :param higher_percent: 最高百分比 - :return: - """ - out = np.zeros_like(bands, dtype=np.uint8) - n = bands.shape[0] - for i in range(n): - a = 0 # np.min(band) - b = 255 # np.max(band) - c = np.percentile(bands[i, :, :], lower_percent) - d = np.percentile(bands[i, :, :], higher_percent) - t = a + (bands[i, :, :] - c) * (b - a) / (d - c) - t[t < a] = a - t[t > b] = b - out[i, :, :] = t - return out - - -def createXML(metadata, xlm_file): - """ - 创建xlm文件并写入字典 - :param metadata: 元数据信息 - :param xlm_file: xlm文件 - :return: - """ - # 创建一个空的文档 - document = minidom.Document() # 创建DOM文档对象 - # 创建一个根节点对象 - root = document.createElement('ProductMetaData') - # 设置根节点的属性 - # root.setAttribute('', '') - # 将根节点添加到文档对象中 - document.appendChild(root) - # 字典转xml - for key in metadata: - # 创建父节点 - node_name = document.createElement(key) - # 给父节点设置文本 - node_name.appendChild(document.createTextNode(str(metadata[key]))) - # 将各父节点添加到根节点 - root.appendChild(node_name) - # 写入xlm文档 - with open(xlm_file, 'w', encoding='utf-8') as f: - document.writexml(f, indent='\t', newl='\n', addindent='\t', encoding='utf-8') - f.close() - - -def GetJPSSData(in_file, xml_path, thumbnail_path): - """ - 获取联合极轨卫星系统（JPSS-1）元数据：NOAA-20(Joint Polar Satellite System spacecraft) - :param xml_path: - :param thumbnail_path: - :param in_file: - :return: 元数据字典 - """ - try: - # 生成缩略图 - in_path, basename = os.path.split(in_file) - ThumbnailName = os.path.splitext(basename)[0] + "_thumb.jpg" - ThumbnailPath = os.path.join(thumbnail_path, ThumbnailName) - - gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES") - in_datasets = gdal.Open(in_file) - meta_data = in_datasets.GetMetadata() - # 取出子数据集 - datasets = in_datasets.GetSubDatasets() - red_data = gdal.Open(datasets[0][0]).ReadAsArray() - nir_data = gdal.Open(datasets[3][0]).ReadAsArray() - swir_data = gdal.Open(datasets[9][0]).ReadAsArray() - img_data = np.array([red_data, nir_data, swir_data]) - img_data = uint16to8(img_data) - # Array转Image - img_data2 = np.transpose(img_data, (1, 2, 0)) - img_data2 = img_data2[:, :, ::-1] - img = Image.fromarray(img_data2) - # 压缩图片大小 - if img_data.shape[1] > img_data.shape[2]: - width = 512 - height = int(width / img_data.shape[1] * img_data.shape[2]) - else: - height = 512 - width = int(height / img_data.shape[1] * img_data.shape[2]) - img.thumbnail((width, height)) - img.save(ThumbnailPath, "PNG") - - # 释放内存 - del in_datasets - del img_data - del img_data2 - del img - - # 生成XML文件 - xmlFileName = os.path.splitext(basename)[0] + ".xml" - xmlPath = os.path.join(xml_path, xmlFileName) - createXML(meta_data, xmlPath) - - # 产品日期 - ProductionTime = meta_data['ProductionTime'] - StartTime = meta_data['StartTime'] - EndTime = meta_data['EndTime'] - - # 其他信息 - ImageGSD = str(meta_data['LongName']).split(" ")[-1] - Bands = str(meta_data['title']).split(" ")[1] - - # 中心经纬度 - productUpperLeftLat = meta_data['NorthBoundingCoordinate'] # 左上纬度 - productUpperLeftLong = meta_data['WestBoundingCoordinate'] # 左上经度 - productUpperRightLat = meta_data['NorthBoundingCoordinate'] # 右上纬度 - productUpperRightLong = meta_data['EastBoundingCoordinate'] # 右上经度 - productLowerLeftLat = meta_data['SouthBoundingCoordinate'] # 左下纬度 - productLowerLeftLong = meta_data['WestBoundingCoordinate'] # 左下经度 - productLowerRightLat = meta_data['SouthBoundingCoordinate'] # 右下纬度 - productLowerRightLong = meta_data['EastBoundingCoordinate'] # 右下纬度 - - # 边界几何 - boundaryGeomStr = f'POLYGON(({productUpperLeftLong} {productUpperLeftLat},' \ - f'{productUpperRightLong} {productUpperRightLat},' \ - f'{productLowerRightLong} {productLowerRightLat},' \ - f'{productLowerLeftLong} {productLowerLeftLat},' \ - f'{productUpperLeftLong} {productUpperLeftLat}))' - - # 构建字典 - jpss_dict = {"ProduceTime": ProductionTime, - "StartTime": StartTime, - "EndTime": EndTime, - "CloudPercent": "", - # "TopLeftLatitude": productUpperLeftLat, - # "TopLeftLongitude": productUpperLeftLong, - # "TopRightLatitude": productUpperRightLat, - # "TopRightLongitude": productUpperRightLong, - # "BottomLeftLatitude": productLowerLeftLat, - # "BottomLeftLongitude": productLowerLeftLong, - # "BottomRightLatitude": productLowerRightLat, - # "BottomRightLongitude": productLowerRightLong, - "boundaryGeomStr": boundaryGeomStr, - "bands": Bands, - "ImageGSD": ImageGSD, - "ProjectedCoordinates": "", - "CollectionCode": "", - "ThumbnailPath": ThumbnailPath, - "ThumbnailName": ThumbnailName, - "xmlPath": xmlPath, - "xmlFileName": xmlFileName, - "DirectoryDepth": "day"} - - # 判断字典是否为空 - if not jpss_dict: - return {"code": -1, "msg": "没有满足条件的数据字典..."} - print(jpss_dict) - return jpss_dict - except Exception as e: - print(str(e)) - return {"code": -1, "msg": str(e)} \ No newline at end of file diff --git a/scan_data/receive_queue.py b/scan_data/receive_queue.py deleted file mode 100644 index 77cbc2a..0000000 --- a/scan_data/receive_queue.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/python3 -# coding= utf-8 - -import sqlite3 - - -# @param text 文本 -# @return True：是，False：不是 -def __string(text): - return True if isinstance(text, str) else False - - -# 检查文本类型是否为浮点型 -# @param text 文本 -# @return True：是，False：不是 -def __float(text): - if __string(text): - try: - return True if float("{0}".format(text)) else False - except Exception: - return False - else: - return True if isinstance(text, float) else False - - -# 检查文本类型是否为浮点型 -# @param text 文本 -# @return True：是，False：不是 -def __int(text): - if __string(text): - try: - return True if float("{0}".format(text)) else False - except Exception: - return False - else: - return True if isinstance(text, float) else False - - -# 检查文本类型是否为数字型 -# @param text 文本 -# @return True：是，False：不是 -def __number(text): - return True if re.search("[^0-9]", text) == None else False - - -def func_year(s): - print('func_year:', s) - - -def func_month(s): - print('func_month:', s) - - -def create_data_collection_info_table(): - con = sqlite3.connect("../dms_client.db") - cur = con.cursor() - sql = "CREATE TABLE IF NOT EXISTS data_collection_info(id INTEGER PRIMARY KEY,collection_code TEXT,function_name TEXT,describe TEXT)" - cur.execute(sql) - # ①：添加单条数据 - data = "1,'Desire',5,'test'" - cur.execute('INSERT INTO data_collection_info VALUES (%s)' % data) - # ②：添加单条数据 - cur.execute("INSERT INTO data_collection_info values(?,?,?,?)", (6, "zgq", 20, 'test')) - # ③：添加多条数据 - cur.executemany('INSERT INTO data_collection_info VALUES (?,?,?,?)', - [(3, 'name3', 19, 'test'), (4, 'name4', 26, 'test')]) - cur.execute("UPDATE data_collection_info SET collection_code=? WHERE id=?", ('test1', 19)) - con.commit() - # 关闭游标 - cur.close() - # 断开数据库连接 - con.close() - - -# 动态调用函数 -# @param tag 标签名 -# @param text 文本 -# @return True:OK, False:NG -def item_check(func_name, text): - if type and text: - try: - # 调用导入模块中的函数，并传参 - return eval("__{0}".format(func_name))(text) - except Exception: - return False - else: - return False - - -if __name__ == '__main__': - result = item_check("num", 123.23) - print(result) - strs = ['year', 'month'] - for s in strs: - globals().get('func_%s' % s)(s) diff --git a/scheduled_task/queue_storage.py b/scheduled_task/queue_storage.py new file mode 100644 index 0000000..dee5e45 --- /dev/null +++ b/scheduled_task/queue_storage.py @@ -0,0 +1,119 @@ +# coding: utf-8 +# Author：tajochen + +import time + +import requests + +from application.settings import Config +from common.tools.dms import dms_login, dms_task_record +from util.upload_client import upload_file_client + +from util.http_util import httpUtil + + +# 动态调用函数 +# @param tag 标签名 +# @param text 文本 +# @return True:OK, False:NG +def item_check(func_name, text): + if type and text: + try: + # 调用导入模块中的函数，并传参 + return eval("__{0}".format(func_name))(text) + except Exception: + return False + else: + return False + + +# 动态调用遥感函数 +# @param tag 标签名 +# @param text 文本 +# @return True:OK, False:NG +def analysis_remote_sensing_data(func_name, collection_code, path): + """ + 解析JPSS-VJ102元数据 + :return: + """ + print('开始扫描VJ102IMG数据集') + #collectionCode = 'VJ102IMG' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collection_code) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return 1 + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + remote_sensing_data_dic = globals().get('func_%s' % func_name)(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(remote_sensing_data_dic['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collection_code} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(remote_sensing_data_dic['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collection_code} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = remote_sensing_data_dic['CollectionCode'] + DirectoryDepth = remote_sensing_data_dic['DirectoryDepth'] + StartTime = remote_sensing_data_dic['StartTime'] + # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) + uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collection_code) + + StartTime = time.mktime(time.strptime(remote_sensing_data_dic['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(remote_sensing_data_dic['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collection_code, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": remote_sensing_data_dic['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": remote_sensing_data_dic['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collection_code, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + if res_data.json()['status'] == 0: + return 0 + else: + return 2 + + +if __name__ == "__main__": + result = item_check("num", 123.23) + print(result) + strs = ['year', 'month'] + for s in strs: + globals().get('func_%s' % s)(s) diff --git a/upload/__init__.py b/upload/__init__.py deleted file mode 100644 index 00d56c8..0000000 --- a/upload/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/12/3 11:17 -Desc: -""" diff --git a/upload/__pycache__/__init__.cpython-39.pyc b/upload/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 4831f2e6057229b9e3a57cd5250f71b8cbecf57f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 205 zcmYe~<>g`kf~MY;iB3TJF^Gc<7=auIATH(r5-AK(3@MDk44O<;W?YV?B^mie3JO*V z5t(_BnF_)AdFfmsnYpP7Kwza{WME{dZ)l`%tYBzpWoXXjl3JW>#pS2Tc#AzgJ|#an zK7J)b5fji*F!9U4#VRH(Gbc5sJijPArl1mNc3x#^VqR`yOiFHXOld(*equ^Ye0*kJ bW=VX!UP0w84x8Nkl+v73JCGAU12F>t5OXt+ diff --git a/upload/__pycache__/upload_client.cpython-39.pyc b/upload/__pycache__/upload_client.cpython-39.pyc deleted file mode 100644 index 43cb636bdc44bd35fe65b3552b1e351fbc6a23ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2021 zcmaJ?&5ImG6tAlOn4X@G{mgD49tLEU5#7zKNlca?k;Fhi_Fz0{v(WT*)nf0uP{U02 zdV9UEqx74ocQ;G#xnD(z=t0}-?Oxm+>p2mdL|B_a(fS?DuJDB>ZyU zDA$joSopchr0~aq8u}9v$nxA-ZIY#GSgxH_^I&rF-cFI`N*19iq~JaeRInpOk|{7N z(mYDsN3U}bW)03LAjF`J>*n%f$19XFikF70Nq2?eWT-!wzZ`V4IVtQAZeO#|B_nP&HDc8RcGEbb$N5z(k^$w z+N=6JZ~f2twqm+nAz8t3+ES|!w>HGBqdW68-Kt{N($m-zC})^tfPK_0B^_t{Bn zaBX*caOLXu?)J6lgLesuFt<*uLLGf{6)5kBG|V^uMe7}`G(fY-SoF^WGqZ3cRM`l#G!-FGbS!JyDv3!Peg{_b(##bQx{)hC zN4sWa!#NC~$e?vhONn$U&0z*>%utEd1wIie58}AA4kt?FvWYtg0!#`qlAAa+Vv$z# zl;>gDUzS^_wYV6oq30ltLhLB+|U9wEeZR;rl!beP6zhN-PpkiiG?SCk;R|jHAOU=H7atqIkPbiGTmmRIz#! zCAm|5$0J$QCdaRF_?~n7&KIBix9)r?2Y3%w&^p%mq_=#4#ahw1#$A{L)w`sN!T TObcP)ug%t3+cXW=cAS3zO~DY` diff --git a/util/UDP_Receive.py b/util/UDP_Receive.py index 5fa56f2..46e8dfc 100644 --- a/util/UDP_Receive.py +++ b/util/UDP_Receive.py @@ -4,12 +4,17 @@ import socket import sys import struct +import json # 本机信息 +import time + +from util.simple_sqlite3_tool import SimpleSQLite3Tool + host_ip = socket.gethostbyname(socket.gethostname()) # 组播组IP和端口 -mcast_group_ip = '239.255.255.252' -mcast_group_port = 5678 +mcast_group_ip = '224.1.1.1' +mcast_group_port = 2234 def receiver(): @@ -34,6 +39,23 @@ def receiver(): while True: try: data, address = sock.recvfrom(4096) + sql_tool = SimpleSQLite3Tool("../dms_client.db") + data2 = json.loads(data) + # 若组播数据不正确 + if len(data2) < 3: + return + # 若数据已存在 + res1 = sql_tool.query("select * from data_list where collection_code=? and file_name=?;", (data2.collectionCode,data2.filename)) + if len(res1) != 0: + return + print(res1) + # 存入归档数据 + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + res2 = sql_tool.execute("insert into data_list " + "(file_path,file_name,collection_code,archive_status,created_time) values (?,?,?,?,?);", + (data2.file_path, data2.file_name, data2.collection_code, 0, current_time)) + print(res2) + sql_tool.close() except socket.error as e: print(f"while receive message error occur:{e}") else: diff --git a/util/__pycache__/file_store_path.cpython-39.pyc b/util/__pycache__/file_store_path.cpython-39.pyc deleted file mode 100644 index a07dca298bb20ca600e13626a318de2194ae6f1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1650 zcmb7^&2JMq6u|BAOfs1?rEJT>E|)owLxoDyvMew{d}vvT3tS2kj3N!Ac-T&vFL^vd zsnSz9RN6DD1glkY0IAZlf?b3_h=0T!AuatEIPvTN4XF^|$bQeU{pP)&-?Lw===H6@8lD53gowJ*69BUh#Lzh}uiVux|G@<_nJ-+}0M;v#+{VBAH zSMdtwT9Mq}J2H$x?9!%vs8v%x8XVbMVf~ zxgTeay}AB$d;Q7g+W%XRU-bX}b1)rjW&tl);;2dZ793n2|J|9TKKEyfEJ}UKWB}W0 z&_J6u1ql-_43-uN6}tS4s6=VYkCdvVxzID8FA9Ar4HH3f&ZdcL?YJXIlHL-SZy+`f z2Hzb!?=)|OQP8}VvYXAU4WyHH;U}@*q;cLXco?a52qtBLCv6|e+G3A9GxT2-Y~>{w zkTHSBF`EFbC4qlzGHc4v75qb>aMV@sd)f-&+7jvN9o*5l(IHe*Uq1qFYj#AIB;1Y4Q3#bgdKs0_qpp)eWxhVIrr`=DDL@=Hv#(ja_!(?SqwE!b9aT%zgy93+tzxJ_4&>!>RzQY*v}87gylW z=YTi=TKj&$yMeO3U|At#GIiMxa+s3CNZk8IuFK>7B- z6#`YpJp*#gC=xh6f?)mr_ahk1v8}tezf|j^Y-FW;8PMb?AnLad!-*X{2Vf>l&x`#q z@x0|pcoG{kjbCRPC+x;a=jc(Vaa?_Jx+cn=N7I()2@?`F*hy%ai#8yZte# zt}&UV2h<5FA%{JA5}Y~a;57*5zwlOG1lK^kc=zOcHJcA(s_9p+s$NyS->dg~ajWH9 z(0=>rUi>R)f8gNR=D^@Q{MDBrxD{HQaGH^vhBUWBJ7*yyIA>>0=$PIOUFccX$i2`b zmT-5PQ(r91NEjUIPiA1QW#-zV#hn=q7kQg|ym_01OWfxRx2^C94|oguqujn~br<)} z`4@(&U&>Brqw{%MTuVDw%i_BKX_|`;w2jW0(`VM#*4NkG?5v;NIP{|LOd zy0%{4PBRfzs+1xc#Ht_hc%)aJpB#x;>SNE(=4GM!n_Z&qoWHf#zO=E%V->Fr2U!{O zM+7c-#6ct!?>cN4J{$fI;jhkuP*!bC2uLy^FI!`(NKH9efw{5GX>A|3#!S&2d&+7y zBM@2(VB4Fr0ekn{)++C;u67;mY{R8W+i3x!8dch@6fcKLJ2z#j1X$R`Foz`&OvXxt z9?A$OnG*Tno+XdKk^h{#xUq#mw{DhlcWW?$*cPKRr$ABG#2OPo4M{VhV_K6bt?AUBS%?-i zHnX57Q)h87T#*L5WZQnPX(3hfRD5O@K&M9j65WMw6EvoCJ(^tz}NyfQ1|TxgPq zZfyKEQ0L+(p5ws-nuS+kR2c#gm-wWO7+!(71bu$o7C}Lj2E#XiPFYj%e@1pFyp?6% z3Pxi`Irz3?NdIv|xqy%YLa+9wjk;mb0Zn!Rj`C`%nzcQ#W8dswQ2vZmJvcW&4e@gZ zmUmU_g_d{!W9fk9ebs)U<^89Y=B7pE*Fdy!6MQ_pyKc^i{2lylau@tvDEy~--Kg2F zwNGRSD3&;&7zcYN>xJo&Ixv3v>S!HI$lKP~uY-M3b=30L(x7pt~6QS((I{DQDW@VAl^lBnZ~Ta49K^EE5TCj*^m#ux&8)MovIDFclmYL2;2nw>t0MdSz_Dnt)@VWu-WU7i6 z%}ZevdY7+!bSb)c<&)6+;KEg-$u&IV4HQ@k=b372j|7{my5f4gQ8}T1vzJF~62OjQVSTLjCq;ZZYR7s`6WRHJ|zH z+xOniym|9x&0H=m!FBm>@0;%pO448XL-(b^!<*2&uLB^dDj^vu4Y?|dUa2b3s}0rG zs+t|E#$*X;D0Wk-#&H70uPbC0C8|l3tfp`rCvj>+0S#qQDyG(7OP}%Sic568qJPtJ zu2}k_>nx`)TQ=6=s_3syy;?4n%cT=~`L)W))9D%P)hg+yFb-+HsIbhc-7u(YAge}M zh9iFhVNKyG(e(=S2cbsK~eVhvq;1N7zx~#HY-u8PqNDDB3c~IJr!@*z><%T8HGc1Ke5zhl3j(9Kdk%;#J zAB}hc_@RjR10Rd{0Pw>RKLGql#18`3BR&XxJmN#ZkD}pV6y`Y+90D8-#sohs_z}T% z!N&zZD)V&vsg$ z-|Th>IlExem5w`I-~Ht;+CSfH-Fg7Z@Tz0j`t-xr_kQ(-=9gD|`>JDFk&|xoCl5b) za_4^Q?&tr*3BlASPDKo8eehhR2kBuI9;;reI+lNxp?%U^bzbGT0DOlnadVb=Rx z0Cnkw#2pe=dPT8dwv1} zqCb<5tt-Xi`bi$+3zrs+bLH~s$y3Js@spFs4PG{+A5NlZoNTCqS<5J*tHiaVQC(?S(n%*6xew%$Kqm@X5uxnk88Yi<)lIBOA)p0ylw=G&Nn5gUMoxrS>LqiNSA z$Z|}IJ(jxU-spUR^-Q~s1}IprGws4E$K%a+wALb6BiNELik7A{LB{#cD8l%3?Rfq~a@LKGnDg2@H=C)A8yM zimF;89@7Xm5mRk*6~JNZ&nW{g*vvbsk2X;gfLrS{53H3?%D1tbzS zVe>Nud1Rcg#W05v4x=0naTo)r_FuyFC@>4W5Mh{3^?5iUS~IwAITnvwHFg!&sa(zO zMgO9?+WllX?=;hGIp}?V^!w^6<8xzG6dTXYxoonz#xl$G6Gqdvc){WW@xBWnB{g|KNy?fW@4mIPoCIl7 z8IgPCqcUfr*Rt{#-#xPOZ;dxa`Il&_@=uL7P5B3J$%OE)Dl)uZ-{(o&}oxA4Z69D*ylh diff --git a/util/__pycache__/scan_file_util.cpython-39.pyc b/util/__pycache__/scan_file_util.cpython-39.pyc deleted file mode 100644 index 0e3ac93a4b8b740fe29e82c615ee8a1055f47da1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23608 zcmeHPYj6}texKR5R{PQ$@stDt1Zbs|u#Huma~SbFz{HDv_5^OTY%j2|579fz8QpAx z4cKrthO=YG=c8=6BzGo`?W?QY*>!#-l@Ixvd`s2jTfzz}UoMYwm5Wqe^85GfbF^6E zAswn>x8^@R{hFTcp8oxR|L*Ci*Xx$>yY^4lqL2Q!B>g*GRR1iv_%@F0!?GkXNs*Y$ zEJ<0B#o3}*#M!D?akeIHDZ65qDUL1aNI4Z}%B8qc4N8OXvnSoDMx`<3Q9LQH;!XJ! zU&^oeQvoHAYEqg~%}R5sMQIUn9m&>Io6;uS&SZP4L+KE1S8_qBQ|T1$hUCK3B4v?q zyOWDkOOz$T-I!dOTBa-$Zcnl+)va{PQoQ@Lr1UT^^PQ2D<;>3lIImz$tQqHC*1}qG zUdh^6JI4#tYozO$MEco8S2&YC?B171#k=qu=^7dw3Jrup z1H)a9jtmaEN8{PphZlT(dmk41Pqmf=hf zkViJ2&L+}_BP^;$`x)zerQC#a@aW%Yqbf>s<<8u4p2H_6C9ge5fL^_@f zP4O8LKFw8ehM(9~$z!LHg7Qx*mpv#?m!=f0tY|f1(Gf@^flt{)S zv19@_hOf8iTlR1rQPr`h)I^eUIDBY*WGt143=W3qk|knlM9oB=OHpXkav?LxVeSe@Qxoe$VV@T$+7ULcR{o zfqbRxIP!Ifd@cM%)yd$dz8RL7Gw;OV!qI@ktvN=eOY(t#&Nr&|qpm9rnxr*ewd6hG z%B^{>%9kvBgt_uw){yt9jha{UownrtBHTmaSG>%96;IG9Ar8VjBfO*v>&Bx%Akn>ISHn674mVC3?e6&RiXw4eliwo9#OI2(?Yw47xP0_0PSu5_| zhI1Rv?KrpN+@ZE=9<5#LIBm@@P}{DwaU~9cLDBX_;#ubP=-#{sdZ?b z+5)z)Q-Vb8LoO0>F{Ga@fo!?5@T$a?!qr-GF(<;7A-Ar}9!+N57pzwnUB!E;In~9; z0clUms2`{1WIf{SGIEF8rMZNk3oW+%$`Z5$`U>igT4rgnExw1a6;)x&G!Ml=9=*8d z%3~J(_Hq2hGjy?4cnTNpiLA;JysPlVpBK*k;>O!&Cf@q!`ln|L zpTAx>f8ooIKPr52=K3culmDA9UH|mgg-aKH``IhkKedSor*n z>z{m3j)Mq=pS>{g&gIF=r@s8z^G41S=TA*sK5t}V_)ngCt?>D4qC9&zy?VNfUhX~7 zRJ`U*Bs{UR3+IZDUdO2(E36Fvd>VW_9nrBe~SUbTpCNi%{LSIg?DrV`?Ik-kf1^-8Y&5 zYN;81ax^}M@P@D&0TG+>&M#rZgIrBTArKQ|uwV zfo>O}5)_@;#CTlyY))o!Y%fp%56xGh?R5XaOfJo${N%QHCKXosHIMF#)f7r;$sWmM zfPjbTUPghkN>jBbl_RS->7KjUL%basE25v%LmI9y?yq=bbpOqn6LF53S6WhWHM%2x zD8o}xs)DUZuXxo`4bi3UM1@EBu`GWGnO?K%)*MgrJ}S9g%^ZuTZ_4X#`hGB&hq92! zc2NnsZ+gqu=(gRuO+7h=7R?;fod$%4uGtm0;iOvUk5GC2SyVepMR%na36EXzpwg2k z-+-vb?Y)NTk{B07*IlG?P;S?(e1wYU7>n{~Dyut>#G|-Jwy%d1%yN3mb!#%7=5`S+ zIyM$hGZYU|>Agacb^91klp4r(Gy|>1k;u}saT*_V*TE?C84Mx1Jv)|6sJxXjrXd6! zXlpp~)ZSfBY#NO`_2e$ymO<{ii^q@W;#oC|PMN_|G*MKWissZEsbk{og7+A?cryk1 zpWG7Ov1eOkbkn{~lo4M+zO9rxlFm@ail!n*;v7$=Tai?E7_F$=6Ipr!7d;A%8oF0x zv2RoOJEA~M;&RKL(MZXwZ;3G`ir7xks1X#D>Qi^odrDd9R=9AP;KrrQbU#%HR2?;% zjELS{?ICK%iT9a$Ag9qxcg1pCjCV%IH{KA6r??PLyeo>(ri!|vXhyz9`$+^Tt|b@aG%D&=j`zJmXreKCw;e`D5Cb${MDZ7sH~_H_NA?ng zWOrCxmR8w@zYG6>?6x$^t#FZ#@poH%vX9&rx7;p!EN-g<*KSL|;*gugnNqqC(k{0n z%qCml?~&d3yKdT?mapCJfb6n(a8OP@ii_MFGUEOpV*k!<^;nUE#q~Q;x)>F#^s7-B zNpWOy9%>_lf(Yf!O3EWtNDHZqP!KJ7D2U7kC2|84Mk|y(JCw)`3_vE7NLw)$R7ask z+M!0;H9K<@)ySQyZ9% z71P3c3>7oqiQ4QGv6qvQ38l~ll~d#;R8CSaR|qw;0lv5!+zncT@M}Qp_Cl?MLSJr= zrN#Dm2VpC#!j>6rhCEiG46CVCkK-?%sf+dDsTy!6H(LXhvr@W}%89Z;qxY@w*{QsrRei8>re2+u~;@T4a$K)o(het^qBoQOQNW@7TB5|0+ z5fY-!MBN{yOR7%ATQXjt3&hk4V)7(KNs&mCsKrR0AwOy;(oC{KvF2mscbo(#A-Y`| zC3RnIZ##)7+1DT>I{_uY%i=D=CO~cqY!bA&<;7NlI-kV@P;AE808Bayz_hemmdZ=( z17<_OGGoZhqf}Ir_*U7Yb*F$>Hf?l4mKMN}`8^ z7;=}BOAPEea`lq96%O(f6gpQp$iGi_`vHkt4%gzT{}oXTe&e{k;d<5w>pw-wZar9g zFl5ie!P;`bhQYc~^FX8b5En4<94|CtUmc!PJ!ro;b z)*`r0tBLEhRdF376?n^xQW4i7ZbMuLrfVO_3+Sa^03#uiQCI$$rRP&3yd zvw4Z@km0<8jO7NO(G0HBUFJG)39i$F^K!v;R!reKFnd>V9pqK!IxFjN9r7ctv#N^E zP}`Ka&R-*JbyZk}>%dgb)|j}?+A`N!C%8_jbS18XvVrULo48J}$aMy4xXxgXbZ27# zfA!cQ=^P$6MsG2W-@ijRtdh4wW%zUS zh@^cRbZVSj71rXQ%UNSBefNsBxW&}(9a0vq(L;VSTWOlohrWgWb=!nx1qh3QfS%bT zEHB`JZjFKS+QxTx7+BB7=NGsaz&Lw5xM!q#fO~g`dt~GLJ>lM0+whW1!M)+a#gDOS zPORCME!^Y(li&XezLPpdJ?vnL-g~mcmRM4XS8N zkr83+pb0RPlgOe$8$l9_Q)EOJX^IR}O>BjrjlEN7q(j+&bOR=&8!RGSs0QhV%m&iJ>+coL{wf5bI741YkW`J2 z<*yGaDA^beZHnypZe-h&qub{KDo>Q5a-|sL%0-$UF!>J&!3Z#K+JfVWu{)rIFHsKX zNSr4jV3JnaiD@p~6;?B21g5jR=HuR?CQV+>14F5=x}|3eIGd;Tspx$RpS1qz0esF& z_?!t*#Jryg49TaQk}aJ=6aj>$6y#%0VTMlwG7b17a@Zk&Q;@+Td;% z5%9Tc3Vg1vflq*N89w{!z$f_;d{#rG89vw6h0k>;OTP&|gJt*}5b#+lT?sx>Ho)hg z2|hzb_#CQ%&*A#;Idu2%IaGm9Yk27Kxq!`oxdm(vO%I!sus9VBsjrZ$hs15I1~H+g zfZZ0dbY@Q*cS zR+tRM`gY^1g5TqeBap4QhDMlMX;sZ)wq#loTC@6QYRX%pX!X4aMa%aprkGzS`X4~p z`wB&Aw(>4X=x_x^%iLf%^2DyqJLUpVH(;PJOt!0QI0HVqpB_>bYk2r^evu$V*l+I; zR|WH|u+a*xiba?64pN2X`TjTG21-QKXzJMl)z*65YB_pc_Rr97ZgM4X23sI}=xyY+bNY~UMq~u40v>GnWgmhh9 zLfVfq1Wkl=piD>y1tBe!u0%+U{UkyrLON6=q{B6YbiJ97-gxz+!cR}%hCSE3&Zm+= z8B51)+jsVd<^na=ldG`GJ_lz58YzF^gSl;w})gKG(!YB===)l)zMX%oWgkp}(teWvX5yojk1S_>5|qDAgQHk2OFC)wYM`h0>>V63eb-*}nhO+E zY$%vmG}~>M%CME`^f1Uz(enta>z#euuK0{qv$P49SnE^I06UNAqmSz6{s%*`8E$d& z+F3+HZ%x}@oD;d{q_6N&8_`aGXc)0k zMMcoYR=R6r3TyI{@yGCel9b&5VY7?<@32Q+tGo`f>ThvaT@lwo@sGNU|3RE1uBf}| F{{dWfd+`7O diff --git a/util/file_store_path.py b/util/file_store_path.py deleted file mode 100644 index 8afd439..0000000 --- a/util/file_store_path.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/4 9:27 -Desc: -""" -import datetime -import os -import time - - -def file_store_path(time_stamp): - """ - :param time_stamp: 时间戳类型时间 - :return: - """ - now = int(round(time_stamp * 1000)) - t = time.localtime(now / 1000) - return os.path.join('E:/data/upload', str(t[0]), str(t[1]), str(t[2])) - - -def file_store_path_year(data_str_time, upload_path, conllection_code): - """ - 目录到年 - :param upload_path: - :param data_str_time: 字符串类型 - :return: - """ - t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') - return os.path.join(upload_path, conllection_code, str(t[0])) - - -def file_store_path_month(data_str_time, upload_path, conllection_code): - """ - 目录到月 - :param upload_path: - :param data_str_time: - :return: - """ - t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') - return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1])) - - -def file_store_path_day(data_str_time, upload_path, conllection_code): - """ - 目录到日 - :param upload_path: - :param data_str_time: 字符串类型的时间 - :return: - """ - t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') - return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1]), str(t[2])) - - -if __name__ == '__main__': - # time_stamp1 = time.time() - # print(time_stamp1) - str_time = '2020-06-08 09:33:07' - t = time.strptime(str_time, '%Y-%m-%d %H:%M:%S') - # path = os.path.join('../upload', str(t[0]), str(t[1]), str(t[2])) - # if not os.path.exists(path): - # os.makedirs(path) - # print(path) - # time_stamp = float(time_stamp1) - # now = int(round(time_stamp * 1000)) - # t = time.localtime(now / 1000) - # print(t) - - # list1 = ['张三', '李四'] - # token_s = dms_login() - # dms_list = dms_sensing_data(token_s) - # 数据库返回值 - # list2 = ['张三', '李四'] - - # d = [y for y in list2 if y not in list1] - # if d is None or len(d) == 0: - # print("d为空") - # else: - # print(d) - # file_dir = 'C:/Users/HP/Desktop/数管/' - # dir_list = os.listdir(file_dir) - # print(dir_list) - # timestring = '2016-12-21 10:22:56' - # print(time.mktime(time.strptime(timestring, '%Y-%m-%d %H:%M:%S'))) # 1482286976.0 - print(t) diff --git a/util/file_util.py b/util/file_util.py deleted file mode 100644 index 4559a7a..0000000 --- a/util/file_util.py +++ /dev/null @@ -1,1050 +0,0 @@ -""" -Author : XinYi Song -Time : 2022/2/12 17:13 -Desc: -""" -import os -import time - -import requests - -from application.settings import Config -from common.tools.dms import dms_login, dms_task_record, dms_sensing_data -from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \ - GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData -from scan_data.example import GetJPSSData -from upload.upload_client import upload_file_client -from util.http_file_upload import upload_client -from util.http_util import httpUtil -from util.remote_sensing_util import gf4_pmi_001 -from util.xml_util import dict_to_xml - - -def list_dir(file_dir): - """ - 通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归 - """ - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = '' # 解析出的xml文件保存的路径 - ThumbnailPath = '' # 解析出的缩略图保存的路径 - for cur_file in d: - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' - xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - # 解析遥感数据文件（demo） - gf4_sensing = gf4_pmi_001(file_name, xml_path) - file_name = os.path.basename(path).split('.')[0] - xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml' - # 将遥感数据写入xml中，并保存本地文件夹中 - dict_to_xml(gf4_sensing, xml_path) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(xml_path, 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = gf4_sensing['CollectionCode'] - DirectoryDepth = gf4_sensing['DirectoryDepth'] - StartTime = gf4_sensing['StartTime'] - uc = upload_client(path, DirectoryDepth, StartTime) - - StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_VJ102_dir(collectionCode, path): - """ - 解析JPSS-VJ102元数据 - :return: - """ - print('开始扫描VJ102IMG数据集') - #collectionCode = 'VJ102IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_VJ103_dir(collectionCode, path): - """ - 解析JPSS-VJ103元数据 - :param file_dir: - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' - print('开始扫描VJ103IMG数据集') - #collectionCode = 'VJ103IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_GF1_PMS2_dir(collectionCode, path): - """ - 获取高分 PMS卫星元数据 - :return: - """ - - print('开始扫描GF1_PMS2_001数据集') - #collectionCode = 'GF1_PMS2_001' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - print(GFPMS_dict['xmlPath']) - files = {'file': open(GFPMS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GFPMS_dict['CollectionCode'] - DirectoryDepth = GFPMS_dict['DirectoryDepth'] - StartTime = GFPMS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_GF3MDJ_dir(collectionCode, path): - """ - 获取高分3号MDJ（GF-3 MDJ）卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描GF3_MDJ_SS数据集') - #collectionCode = 'GF3_MDJ_SS' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GF3_MDJ_SS_dict['CollectionCode'] - DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth'] - StartTime = GF3_MDJ_SS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_H08_dir(collectionCode, path): - """ - 获取葵花8卫星元数据 - :return: - """ - print('开始扫描H08数据集') - #collectionCode = 'NC_H08' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetH08_dict['CollectionCode'] - DirectoryDepth = GetH08_dict['DirectoryDepth'] - StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_Sentinel1_dir(collectionCode, path): - """ - 获取哨兵1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描Sentinel1数据集') - #collectionCode = 'S1A_IW_GRDH' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel1_dict['CollectionCode'] - DirectoryDepth = Sentinel1_dict['DirectoryDepth'] - StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_Sentinel2_dir(collectionCode, path): - """ - 获取哨兵2号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描Sentinel2数据集') - #collectionCode = 'S2B' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel2_dict['CollectionCode'] - DirectoryDepth = Sentinel2_dict['DirectoryDepth'] - StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_Sentinel3OL_dir(collectionCode, path): - """ - 获取哨兵3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描Sentinel3数据集') - #collectionCode = 'Sentinel3_OLCI' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath) - print(Sentinel3OL_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel3OL_dict['CollectionCode'] - DirectoryDepth = Sentinel3OL_dict['DirectoryDepth'] - StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_HJ1_dir(collectionCode, path): - """ - 获取环境1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1' - print('开始扫描环境1号数据集') - #collectionCode = 'HJ-1' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetHJ1Data_dict['CollectionCode'] - DirectoryDepth = GetHJ1Data_dict['DirectoryDepth'] - StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_ZY3_dir(collectionCode, path): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3' - print('开始扫描资源3号数据集') - #collectionCode = 'ZY-3' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetZY3Data_dict['CollectionCode'] - DirectoryDepth = GetZY3Data_dict['DirectoryDepth'] - StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19] - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_SNPP_dir(collectionCode, path): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS' - print('开始扫描VNP02IMG数据集') - #collectionCode = 'VNP02IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath) - print(GetSNPPData_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetSNPPData_dict['CollectionCode'] - DirectoryDepth = GetSNPPData_dict['DirectoryDepth'] - StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime, collectionCode) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -if __name__ == '__main__': - # file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata' - # list_dir(file_dir) - # file_dir = 'E:/数管' - # - scan_VJ102_dir() - # - # scan_VJ103_dir() - - # scan_GF1_PMS2_dir() - # scan_GF3MDJ_dir() - # scan_H08_dir() - # scan_Sentinel1_dir() - # scan_Sentinel2_dir() - # scan_Sentinel3OL_dir() - # scan_HJ1_dir() - # scan_ZY3_dir() - # scan_SNPP_dir() diff --git a/util/find_GPS_image.py b/util/find_GPS_image.py deleted file mode 100644 index 7b144e6..0000000 --- a/util/find_GPS_image.py +++ /dev/null @@ -1,81 +0,0 @@ -import exifread -import re -import json -import requests - -def latitude_and_longitude_convert_to_decimal_system(*arg): - """ - 经纬度转为小数, param arg: - :return: 十进制小数 - """ - return float(arg[0]) + ((float(arg[1]) + (float(arg[2].split('/')[0]) / float(arg[2].split('/')[-1]) / 60)) / 60) - -def find_GPS_image(pic_path): - GPS = {} - date = '' - with open(pic_path, 'rb') as f: - tags = exifread.process_file(f) - for tag, value in tags.items(): - if re.match('GPS GPSLatitudeRef', tag): - GPS['GPSLatitudeRef'] = str(value) - elif re.match('GPS GPSLongitudeRef', tag): - GPS['GPSLongitudeRef'] = str(value) - elif re.match('GPS GPSAltitudeRef', tag): - GPS['GPSAltitudeRef'] = str(value) - elif re.match('GPS GPSLatitude', tag): - try: - match_result = re.match('\[(\w*),(\w*),(\w.*)/(\w.*)\]', str(value)).groups() - GPS['GPSLatitude'] = int(match_result[0]), int(match_result[1]), int(match_result[2]) - except: - deg, min, sec = [x.replace(' ', '') for x in str(value)[1:-1].split(',')] - GPS['GPSLatitude'] = latitude_and_longitude_convert_to_decimal_system(deg, min, sec) - elif re.match('GPS GPSLongitude', tag): - try: - match_result = re.match('\[(\w*),(\w*),(\w.*)/(\w.*)\]', str(value)).groups() - GPS['GPSLongitude'] = int(match_result[0]), int(match_result[1]), int(match_result[2]) - except: - deg, min, sec = [x.replace(' ', '') for x in str(value)[1:-1].split(',')] - GPS['GPSLongitude'] = latitude_and_longitude_convert_to_decimal_system(deg, min, sec) - elif re.match('GPS GPSAltitude', tag): - GPS['GPSAltitude'] = str(value) - elif re.match('.*Date.*', tag): - date = str(value) - return {'GPS_information': GPS, 'date_information': date} - -def find_address_from_GPS(GPS): - """ - 使用Geocoding API把经纬度坐标转换为结构化地址。 - :param GPS: - :return: - """ - secret_key = 'zbLsuDDL4CS2U0M4KezOZZbGUY9iWtVf' - if not GPS['GPS_information']: - return '该照片无GPS信息' - lat, lng = GPS['GPS_information']['GPSLatitude'], GPS['GPS_information']['GPSLongitude'] - baidu_map_api = "http://api.map.baidu.com/geocoder/v2/?ak={0}&callback=renderReverse&location={1},{2}s&output=json&pois=0".format( - secret_key, lat, lng) - response = requests.get(baidu_map_api) - content = response.text.replace("renderReverse&&renderReverse(", "")[:-1] - baidu_map_address = json.loads(content) - formatted_address = baidu_map_address["result"]["formatted_address"] - province = baidu_map_address["result"]["addressComponent"]["province"] - city = baidu_map_address["result"]["addressComponent"]["city"] - district = baidu_map_address["result"]["addressComponent"]["district"] - return formatted_address,province,city,district - -pic_path = 'D:\pythonjob\pic-time&location\DJI_0001.jpg' - -GPS_info = find_GPS_image(pic_path) -address = find_address_from_GPS(GPS=GPS_info) -#print(GPS_info) -#print(address) - -x = list(GPS_info.values()) -#print(x) -time = x[1] -gps_dict_formate = x[0] -y = list(gps_dict_formate.values()) -information = '拍照时间：'+time+'，拍照地点：'+str(address[0])+'（经度:'+str(y[2])+' '+str(y[3])+'，纬度：'+str(y[0])+' '+str(y[1])+'，高度：'+str(y[5])+'米）' - -print(pic_path) -print(information) diff --git a/util/http_file_upload.py b/util/http_file_upload.py deleted file mode 100644 index d2f1a7e..0000000 --- a/util/http_file_upload.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/3 14:29 -Desc: -""" -from util.file_store_path import file_store_path_day, file_store_path_year, file_store_path_month - -""" -实现文件断点续传 -""" -import sys -import os -from hashlib import md5 - -FILE_DIR = os.path.dirname(__file__) - -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) -home = os.path.join(BASE_DIR, "E:/data/upload") - - -# 定义一个函数，计算进度条 -def bar(num=1, sum=100): - rate = float(num) / float(sum) - rate_num = int(rate * 100) - temp = '\r%d %%' % rate_num - sys.stdout.write(temp) - - -def md5_file(name): - m = md5() - a_file = open(name, 'rb') #需要使用二进制格式读取文件内容 - m.update(a_file.read()) - a_file.close() - return m.hexdigest() - - -def upload_client(local_path, depth, dateTime, conllection_code): - global file_path - while True: - file_byte_size = os.stat(local_path).st_size # 获取文件的大小 - file_name = os.path.basename(local_path) # 设置文件名 - md5 = md5_file(local_path) - - has_sent = 0 - file_obj = open(local_path, 'rb') # 对文件进行读操作 - file_obj.seek(has_sent) # 调整指针 - if depth == 'year': - file_path = file_store_path_year(dateTime, home, conllection_code) - if not os.path.exists(file_path): - os.makedirs(file_path) - if depth == 'month': - file_path = file_store_path_month(dateTime, home, conllection_code) - if not os.path.exists(file_path): - os.makedirs(file_path) - if depth == 'day': - file_path = file_store_path_day(dateTime, home, conllection_code) - if not os.path.exists(file_path): - os.makedirs(file_path) - path = os.path.join(file_path, file_name) - has_received = 0 - - # 首先判断该路径下是否已存在文件 - if os.path.exists(path): - f = open(path, 'wb') - else: - f = open(path, 'wb') - - while has_sent < file_byte_size: - # 读出数据 - data = file_obj.read(1024) - try: - # 写入数据 - f.write(data) - has_received += len(data) - if not data: - raise Exception - except Exception: - flag = False - break - has_sent += len(data) - bar(has_sent, file_byte_size) # 进度条 - print("文件上传成功！") - file_obj.close() - f.close() - file_dict = {'fileName': file_name, 'md5': md5, 'file_size': file_byte_size, 'file_path': file_path, 'type': 'ok'} - return file_dict diff --git a/util/remote_sensing_util.py b/util/remote_sensing_util.py deleted file mode 100644 index 18440ff..0000000 --- a/util/remote_sensing_util.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/4 16:59 -Desc: -""" -import rasterio -import requests - -from util.xml_util import xml_to_dict, dict_to_xml - - -def gf4_pmi_001(file_name, xml_name): - """ - - :param file_name: 扫描文件传过来的遥感数据源文件的路径 - :param xmlPath: 解析出的xml文件存储的路径 - :param ThumbnailPath: 解析出的缩略图文件存储的路径 - :return: - """ - file_path = 'E:/sensing/GF4_PMI_001/' - with rasterio.open(file_path+file_name, 'r') as ds: - # 存放xml，缩略图的文件夹，由根文件夹+数据集代码命名的文件夹组成 - print('该栅格数据的基本数据集信息:') - CollectionCode = 'GF4_PMI_001' # 数据集代码 - DataFormat = ds.driver # DataFormat 数据格式 - NumberBands = ds.count # NumberBands 波段数目 - ImageWidth = ds.width # ImageWidth 影像宽度 - ImageHeight = ds.height # ImageHeight 影像高度 - GeographicScope = ds.bounds # GeographicScope 地理范围 - ReflectionParameter = ds.transform # ReflectionParameter 反射变换参数（六参数模型） - ProjectionDefinition = ds.crs # ProjectionDefinition 投影定义 - # print(CRS.from_epsg(4326)) - # 获取第一个波段数据，跟GDAL一样索引从1开始 - # 直接获得numpy.ndarray类型的二维数组表示，如果read()函数不加参数，则得到所有波段（第一个维度是波段） - band1 = ds.read(1) - FirstBindMax = band1.max() # FirstBindMax 第一波段的最大值 - - FirstBindMin = band1.min() # FirstBindMin 第一波段的最小值 - FirstBindAverage = band1.mean() # FirstBindAverage 第一波段的平均值 - # 根据地理坐标得到行列号 - x, y = (ds.bounds.left + 300, ds.bounds.top - 300) # 距离左上角东300米，南300米的投影坐标 - row, col = ds.index(x, y) # 对应的行列号 - print(f'(投影坐标{x}, {y})对应的行列号是({row}, {col})') - ProjectedCoordinates = x, y # ProjectedCoordinates 投影坐标 - RowNumber = row, col # RowNumber 对应的行列号 - # 根据行列号得到地理坐标 - x, y = ds.xy(row, col) # 中心点的坐标 - print(f'行列号({row}, {col})对应的中心投影坐标是({x}, {y})') # - CenterProjectionCoordinates = x, y # CenterProjectionCoordinates 中心投影坐标 - # 'C:/Users/HP/Desktop/Number tube/GF4_PMI_E119.8_N35.3_20210908_L1A0000417337/GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - # 传入xml文件路径，解析xml文件 - # xml_name 存储后的xml文件的路径+文件名 - xml_dict = xml_to_dict(file_path+xml_name) - StartTime = xml_dict['ProductMetaData']['StartTime'] # 开始采集时间 - EndTime = xml_dict['ProductMetaData']['EndTime'] # 结束采集时间 - CloudPercent = xml_dict['ProductMetaData']['CloudPercent'] # 云覆盖量百分比 - TopLeftLatitude = xml_dict['ProductMetaData']['TopLeftLatitude'] # 左上纬度 - TopLeftLongitude = xml_dict['ProductMetaData']['TopLeftLongitude'] # 左上经度 - TopRightLatitude = xml_dict['ProductMetaData']['TopRightLatitude'] # 右上纬度 - TopRightLongitude = xml_dict['ProductMetaData']['TopRightLongitude'] # 右上经度 - BottomRightLatitude = xml_dict['ProductMetaData']['BottomRightLatitude'] # 右下纬度 - BottomRightLongitude = xml_dict['ProductMetaData']['BottomRightLongitude'] # 右下经度 - BottomLeftLatitude = xml_dict['ProductMetaData']['BottomLeftLatitude'] # 左下纬度 - BottomLeftLongitude = xml_dict['ProductMetaData']['BottomLeftLongitude'] # 左下经度 - boundaryGeomStr = f'POLYGON(({TopLeftLatitude} {TopLeftLongitude},' \ - f'{TopRightLatitude} {TopRightLongitude},' \ - f'{BottomRightLatitude} {BottomRightLongitude},' \ - f'{BottomLeftLatitude} {BottomLeftLongitude},' \ - f'{TopLeftLatitude} {TopLeftLongitude}))' - # ThumbnailPath 存储后的缩略图的路径+文件名 ThumbnailName 缩略图文件名称 - # xmlPath 存储后的xml文件路径+文件名 xmlFileName xml文件名称 - sensing_dict = {'StartTime': StartTime, 'EndTime': EndTime, 'CloudPercent': CloudPercent, - 'boundaryGeomStr': boundaryGeomStr, 'DataFormat': DataFormat, 'NumberBands': NumberBands, - 'ImageWidth': ImageWidth, 'ImageHeight': ImageHeight, 'GeographicScope': GeographicScope, - #'ReflectionParameter': ReflectionParameter, 'ProjectionDefinition': ProjectionDefinition, - #'FirstBindMax': FirstBindMax, 'FirstBindMin': FirstBindMin, - 'FirstBindAverage': FirstBindAverage, - 'ProjectedCoordinates': ProjectedCoordinates, 'RowNumber': RowNumber, - #'CenterProjectionCoordinates': CenterProjectionCoordinates, - 'CollectionCode': CollectionCode, - "ThumbnailPath": file_path+"GF4_IRS_E119.8_N35.3_20210908_L1A0000417337_thumb.jpg", - "ThumbnailName": "GF4_IRS_E119.8_N35.3_20210908_L1A0000417337_thumb.jpg", - "xmlPath": "", "xmlFileName": "", - 'DirectoryDepth': 'day'} - return sensing_dict - - -if __name__ == '__main__': - file_path = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_E119.8_N35.3_20210908_L1A0000417337/GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' - xml_path = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_E119.8_N35.3_20210908_L1A0000417337/GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - gf4_pmi_001(file_path, xml_path) diff --git a/util/scan_file_util.py b/util/scan_file_util.py deleted file mode 100644 index 4a4140b..0000000 --- a/util/scan_file_util.py +++ /dev/null @@ -1,1219 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/3 9:00 -Desc: -""" -import os -import time - -import requests - -from application.settings import Config -from common.tools.dms import dms_login, dms_task_record, dms_sensing_data -from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \ - GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData -from scan_data.example import GetJPSSData -from upload.upload_client import upload_file_client -from util.http_file_upload import upload_client -from util.http_util import httpUtil -from util.remote_sensing_util import gf4_pmi_001 -from util.xml_util import dict_to_xml -from apscheduler.schedulers.blocking import BlockingScheduler - -sched = BlockingScheduler() - - -def list_dir(file_dir): - """ - 通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归 - """ - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = '' # 解析出的xml文件保存的路径 - ThumbnailPath = '' # 解析出的缩略图保存的路径 - for cur_file in d: - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' - xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - # 解析遥感数据文件（demo） - gf4_sensing = gf4_pmi_001(file_name, xml_path) - file_name = os.path.basename(path).split('.')[0] - xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml' - # 将遥感数据写入xml中，并保存本地文件夹中 - dict_to_xml(gf4_sensing, xml_path) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(xml_path, 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = gf4_sensing['CollectionCode'] - DirectoryDepth = gf4_sensing['DirectoryDepth'] - StartTime = gf4_sensing['StartTime'] - uc = upload_client(path, DirectoryDepth, StartTime) - - StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_VJ102_dir(): - """ - 解析JPSS-VJ102元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' - dir_list = os.listdir(file_dir) - print('开始扫描VJ102IMG数据集') - collectionCode = 'VJ102IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'VJ102IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - print(path) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_VJ103_dir(): - """ - 解析JPSS-VJ103元数据 - :param file_dir: - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' - print('开始扫描VJ103IMG数据集') - collectionCode = 'VJ103IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'VJ103IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_GF1_PMS2_dir(): - """ - 获取高分 PMS卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\GF-1' - print('开始扫描GF1_PMS2_001数据集') - collectionCode = 'GF1_PMS2_001' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'GF1_PMS2' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - print(GFPMS_dict['xmlPath']) - files = {'file': open(GFPMS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GFPMS_dict['CollectionCode'] - DirectoryDepth = GFPMS_dict['DirectoryDepth'] - StartTime = GFPMS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_GF3MDJ_dir(): - """ - 获取高分3号MDJ（GF-3 MDJ）卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\GF-3' - print('开始扫描GF3_MDJ_SS数据集') - collectionCode = 'GF3_MDJ_SS' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'GF3_MDJ' in cur_file[0:7] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GF3_MDJ_SS_dict['CollectionCode'] - DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth'] - StartTime = GF3_MDJ_SS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_H08_dir(): - """ - 获取葵花8卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\葵花8' - print('开始扫描H08数据集') - collectionCode = 'NC_H08' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'NC_H08' in cur_file[0:6] and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetH08_dict['CollectionCode'] - DirectoryDepth = GetH08_dict['DirectoryDepth'] - StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' '+ GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_Sentinel1_dir(): - """ - 获取哨兵1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\SENTINEL-1' - print('开始扫描Sentinel1数据集') - collectionCode = 'S1A_IW_GRDH' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'S1A_IW_GRDH' in cur_file[0:11] and os.path.splitext(cur_file)[1] == '.zip': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel1_dict['CollectionCode'] - DirectoryDepth = Sentinel1_dict['DirectoryDepth'] - StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_Sentinel2_dir(): - """ - 获取哨兵2号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\SENTINEL-2' - print('开始扫描Sentinel2数据集') - collectionCode = 'S2B' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'S2B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel2_dict['CollectionCode'] - DirectoryDepth = Sentinel2_dict['DirectoryDepth'] - StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_Sentinel3OL_dir(): - """ - 获取哨兵3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\SENTINEL-3' - print('开始扫描Sentinel3数据集') - collectionCode = 'Sentinel3_OLCI' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'S3B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - print(path) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath) - print(Sentinel3OL_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel3OL_dict['CollectionCode'] - DirectoryDepth = Sentinel3OL_dict['DirectoryDepth'] - StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_HJ1_dir(): - """ - 获取环境1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1' - print('开始扫描环境1号数据集') - collectionCode = 'HJ-1' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'HJ1' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetHJ1Data_dict['CollectionCode'] - DirectoryDepth = GetHJ1Data_dict['DirectoryDepth'] - StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_ZY3_dir(): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3' - print('开始扫描资源3号数据集') - collectionCode = 'ZY-3' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'ZY3' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetZY3Data_dict['CollectionCode'] - DirectoryDepth = GetZY3Data_dict['DirectoryDepth'] - StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19] - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_SNPP_dir(): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS' - print('开始扫描VNP02IMG数据集') - collectionCode = 'VNP02IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'VNP02IMG' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath) - print(GetSNPPData_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetSNPPData_dict['CollectionCode'] - DirectoryDepth = GetSNPPData_dict['DirectoryDepth'] - StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime, collectionCode) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -if __name__ == '__main__': - # file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata' - # list_dir(file_dir) - # file_dir = 'E:/数管' - # - scan_VJ102_dir() - # - # scan_VJ103_dir() - - # scan_GF1_PMS2_dir() - # scan_GF3MDJ_dir() - # scan_H08_dir() - # scan_Sentinel1_dir() - # scan_Sentinel2_dir() - # scan_Sentinel3OL_dir() - # scan_HJ1_dir() - # scan_ZY3_dir() - # scan_SNPP_dir() \ No newline at end of file diff --git a/util/simple_sqlite3_tool.py b/util/simple_sqlite3_tool.py index 75d85c5..bd32962 100644 --- a/util/simple_sqlite3_tool.py +++ b/util/simple_sqlite3_tool.py @@ -1,6 +1,5 @@ # coding: utf-8 -# Author：boxker -# Mail：icjb@foxmail.com +# Author：tajochen import sqlite3 import os @@ -13,79 +12,84 @@ class SimpleSQLite3Tool: 编写这个类主要是为了封装sqlite，继承此类复用方法 """ - def __init__(self, filename="stsql"): - """ - 初始化数据库，默认文件名 stsql.db - filename：文件名 - """ - self.filename = filename - self.db = sqlite3.connect(self.filename) - self.c = self.db.cursor() - def close(self): - """ - 关闭数据库 - """ - self.c.close() - self.db.close() +def __init__(self, filename="stsql"): + """ + 初始化数据库，默认文件名 stsql.db + filename：文件名 + """ + self.filename = filename + self.db = sqlite3.connect(self.filename) + self.c = self.db.cursor() - def execute(self, sql, param=None): - """ - 执行数据库的增、删、改 - sql：sql语句 - param：数据，可以是list或tuple，亦可是None - retutn：成功返回True - """ - try: - if param is None: - self.c.execute(sql) - else: - if type(param) is list: - self.c.executemany(sql, param) - else: - self.c.execute(sql, param) - count = self.db.total_changes - self.db.commit() - except Exception as e: - print(e) - return False, e - if count > 0: - return True - else: - return False - def query(self, sql, param=None): - """ - 查询语句 - sql：sql语句 - param：参数,可为None - retutn：成功返回True - """ +def close(self): + """ + 关闭数据库 + """ + self.c.close() + self.db.close() + + +def execute(self, sql, param=None): + """ + 执行数据库的增、删、改 + sql：sql语句 + param：数据，可以是list或tuple，亦可是None + return：成功返回True + """ + try: if param is None: self.c.execute(sql) else: - self.c.execute(sql, param) - return self.c.fetchall() + if type(param) is list: + self.c.executemany(sql, param) + else: + self.c.execute(sql, param) + count = self.db.total_changes + self.db.commit() + except Exception as e: + print(e) + return False, e + if count > 0: + return True + else: + return False - # def set(self,table,field=" * ",where="",isWhere=False): - # self.table = table - # self.filed = field - # if where != "" : - # self.where = where - # self.isWhere = True - # return True + +def query(self, sql, param=None): + """ + 查询语句 + sql：sql语句 + param：参数,可为None + return：成功返回True + """ + if param is None: + self.c.execute(sql) + else: + self.c.execute(sql, param) + return self.c.fetchall() + + +# def set(self,table,field=" * ",where="",isWhere=False): +# self.table = table +# self.filed = field +# if where != "" : +# self.where = where +# self.isWhere = True +# return True if __name__ == "__main__": # 数据库文件位置 - sql = SimpleSQLite3Tool("../dms_client.db") - # f = sql.execute("create table test (id int not null,name text not null,age int);") - # print("ok") - # sql.execute("insert into test (id,name,age) values (?,?,?);", [(1, 'abc', 15), (2, 'bca', 16)]) - # res = sql.query("select * from test;") - # print(res) - # sql.execute("insert into test (id,name) values (?,?);", (3, 'bac')) - # res = sql.query("select * from test where id=?;", (3,)) + sql = SimpleSQLite3Tool("../test.db") + f = sql.execute("create table test (id int not null,name text not null,age int);") + print("ok") + sql.execute("insert into test (id,name,age) values (?,?,?);", [(1, 'abc', 15), (2, 'bca', 16)]) + res = sql.query("select * from test;") + print(res) + sql.execute("insert into test (id,name) values (?,?);", (3, 'bac')) + res = sql.query("select * from test where id=?;", (3,)) res = sql.query("select * from data_collection_info;") print(res) sql.close() diff --git a/upload/upload_client.py b/util/upload_client.py similarity index 69% rename from upload/upload_client.py rename to util/upload_client.py index 72b2de1..f14f6ec 100644 --- a/upload/upload_client.py +++ b/util/upload_client.py @@ -8,8 +8,7 @@ import sys import re import os from hashlib import md5 - -from util.file_store_path import file_store_path_year, file_store_path_month, file_store_path_day +import time FILE_DIR = os.path.dirname(__file__) @@ -26,6 +25,27 @@ def bar(num=1, sum=100): sys.stdout.write(temp) +def file_store_path(time_stamp): + """ + :param time_stamp: 时间戳类型时间 + :return: + """ + now = int(round(time_stamp * 1000)) + t = time.localtime(now / 1000) + return os.path.join('E:/data/upload', str(t[0]), str(t[1]), str(t[2])) + + +def file_store_path_year(data_str_time, upload_path, conllection_code): + """ + 目录到年 + :param upload_path: + :param data_str_time: 字符串类型 + :return: + """ + t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') + return os.path.join(upload_path, conllection_code, str(t[0])) + + def md5_file(name): m = md5() a_file = open(name, 'rb') # 需要使用二进制格式读取文件内容 @@ -34,6 +54,28 @@ def md5_file(name): return m.hexdigest() +def file_store_path_month(data_str_time, upload_path, conllection_code): + """ + 目录到月 + :param upload_path: + :param data_str_time: + :return: + """ + t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') + return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1])) + + +def file_store_path_day(data_str_time, upload_path, conllection_code): + """ + 目录到日 + :param upload_path: + :param data_str_time: 字符串类型的时间 + :return: + """ + t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') + return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1]), str(t[2])) + + def upload_file_client(file_path, depth, dateTime, collectionCode): ck = socket.socket() ck.connect(('192.168.2.9', 9002)) diff --git a/util/zxby.py b/util/zxby.py deleted file mode 100644 index 261eaf7..0000000 --- a/util/zxby.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/10/9 9:43 -Desc: -""" -import os, sys, subprocess, tempfile, time - -# 创建临时文件夹,返回临时文件夹路径 -TempFile = tempfile.mkdtemp(suffix='_test', prefix='python_') -# 文件名 -FileNum = int(time.time() * 1000) -# python编译器位置 -EXEC = sys.executable - - -# 获取python版本 -def get_version(): - v = sys.version_info - version = "python %s.%s" % (v.major, v.minor) - return version - - -# 获得py文件名 -def get_pyname(): - global FileNum - return 'test_%d' % FileNum - - -# 接收代码写入文件 -def write_file(pyname, code): - fpath = os.path.join(TempFile, '%s.py' % pyname) - with open(fpath, 'w', encoding='utf-8') as f: - f.write(code) - print('file path: %s' % fpath) - return fpath - - -# 编码 -def decode(s): - try: - return s.decode('utf-8') - except UnicodeDecodeError: - return s.decode('gbk') - - # 主执行函数 - - -def main(code): - r = dict() - r["version"] = get_version() - pyname = get_pyname() - fpath = write_file(pyname, code) - try: - # subprocess.check_output 是 父进程等待子进程完成，返回子进程向标准输出的输出结果 - # stderr是标准输出的类型 - outdata = decode(subprocess.check_output([EXEC, fpath], stderr=subprocess.STDOUT, timeout=5)) - except subprocess.CalledProcessError as e: - # e.output是错误信息标准输出 - # 错误返回的数据 - r["code"] = 'Error' - r["output"] = decode(e.output) - return r - else: - # 成功返回的数据 - r['output'] = outdata - return r - finally: - # 删除文件(其实不用删除临时文件会自动删除) - try: - os.remove(fpath) - except Exception as e: - exit(1) - - -# if __name__ == '__main__': -# code = "print(11);print(22)" -# print(main(code)) -