config读写配置更新

magic_pdf/libs/config_reader.py

@@ -10,32 +10,27 @@ from loguru import logger
 
 def get_s3_config(bucket_name: str):
     """
-    ~/magic_pdf_config.json 读出来
+    ~/magic-pdf.json 读出来
     """
-    if os.name == "posix":  # Linux or macOS
+
     home_dir = os.path.expanduser("~")
-    elif os.name == "nt":  # Windows
-        home_dir = os.path.expandvars("%USERPROFILE%")
-    else:
-        raise Exception("Unsupported operating system")
 
-    config_file = os.path.join(home_dir, "magic_pdf_config.json")
+    config_file = os.path.join(home_dir, "magic-pdf.json")
 
     if not os.path.exists(config_file):
-        raise Exception("magic_pdf_config.json not found")
+        raise Exception("magic-pdf.json not found")
 
     with open(config_file, "r") as f:
         config = json.load(f)
 
-    if bucket_name not in config:
-        raise Exception("bucket_name not found in magic_pdf_config.json")
+    bucket_info = config.get("bucket_info")
+    if bucket_name not in bucket_info:
+        raise Exception("bucket_name not found in magic-pdf.json")
 
-    access_key = config[bucket_name].get("ak")
-    secret_key = config[bucket_name].get("sk")
-    storage_endpoint = config[bucket_name].get("endpoint")
+    access_key, secret_key, storage_endpoint = bucket_info[bucket_name]
 
     if access_key is None or secret_key is None or storage_endpoint is None:
-        raise Exception("ak, sk or endpoint not found in magic_pdf_config.json")
+        raise Exception("ak, sk or endpoint not found in magic-pdf.json")
 
     # logger.info(f"get_s3_config: ak={access_key}, sk={secret_key}, endpoint={storage_endpoint}")
 

magic_pdf/spark/config_init_to_json.py → utils/config_init_to_json.py

@@ -18,11 +18,7 @@ def get_bucket_configs_dict(buckets, clusters, users):
         # logger.info(bucket_name)
         # logger.info(endpoint)
         # logger.info(user_config)
-        bucket_config = {
-            "endpoint": endpoint,
-            "ak": user_config["ak"],
-            "sk": user_config["sk"],
-        }
+        bucket_config = [user_config["ak"], user_config["sk"], endpoint]
         bucket_configs[bucket_name] = bucket_config
 
     return bucket_configs
@@ -32,16 +28,10 @@ def write_json_to_home(my_dict):
     # Convert dictionary to JSON
     json_data = json.dumps(my_dict, indent=4, ensure_ascii=False)
 
-    # Determine the home directory path based on the operating system
-    if os.name == "posix":  # Linux or macOS
     home_dir = os.path.expanduser("~")
-    elif os.name == "nt":  # Windows
-        home_dir = os.path.expandvars("%USERPROFILE%")
-    else:
-        raise Exception("Unsupported operating system")
 
     # Define the output file path
-    output_file = os.path.join(home_dir, "magic_pdf_config.json")
+    output_file = os.path.join(home_dir, "magic-pdf.json")
 
     # Write JSON data to the output file
     with open(output_file, "w") as f:
@@ -54,4 +44,8 @@ def write_json_to_home(my_dict):
 if __name__ == '__main__':
     bucket_configs_dict = get_bucket_configs_dict(s3_buckets, s3_clusters, s3_users)
     logger.info(bucket_configs_dict)
-    write_json_to_home(bucket_configs_dict)
+    config_dict = {
+        "bucket_info": bucket_configs_dict,
+        "temp-output-dir": "/tmp"
+    }
+    write_json_to_home(config_dict)