Merge remote-tracking branch 'origin/master'

# Conflicts:
#	magic_pdf/model/pdf_extract_kit.py

.github/workflows/benchmark.yml

@@ -18,15 +18,12 @@ on:
   workflow_dispatch:
 jobs:
   pdf-test:
-    runs-on: mineru
+    runs-on: ubuntu-latest
     timeout-minutes: 180
     strategy:
       fail-fast: true
 
     steps:
-    - name: config-net
-      run: |
-          source activate base
     - name: PDF benchmark
       uses: actions/checkout@v3
       with:

.github/workflows/cli.yml

@@ -18,15 +18,12 @@ on:
   workflow_dispatch:
 jobs:
   cli-test:
-    runs-on: mineru
+    runs-on: ubuntu-latest
     timeout-minutes: 40
     strategy:
       fail-fast: true
 
     steps:
-    - name: config-net
-      run: |
-        source activate base
     - name: PDF cli
       uses: actions/checkout@v3
       with:
@@ -34,19 +31,11 @@ jobs:
       
     - name: check-requirements
       run: |
-        changed_files=$(git diff --name-only -r HEAD~1 HEAD)
-        echo $changed_files
-        if [[ $changed_files =~ "requirements.txt" ]] || [[ $changed_files =~ "requirements-qa.txt" ]]; then
         pip install -r requirements.txt
         pip install -r requirements-qa.txt
-        fi
-
-    - name: config-net-reset
-      run: |
-        export http_proxy=""
-        export https_proxy=""
     - name: test_cli
       run: |
+        cp magic-pdf.template.json ~/magic-pdf.json
         echo $GITHUB_WORKSPACE
         cd $GITHUB_WORKSPACE && export PYTHONPATH=. && pytest -s -v tests/test_unit.py
         cd $GITHUB_WORKSPACE &&  pytest -s -v tests/test_cli/test_cli.py

others/README.md

-# pdf_toolbox
-pdf 解析基础函数
-
-
-## pdf是否是文字类型/扫描类型的区分
-
-```shell
-cat s3_pdf_path.example.pdf | parallel --colsep ' ' -j 10 "python pdf_meta_scan.py --s3-pdf-path {2} --s3-profile {1} >> {/}.jsonl"
-
-find dir/to/jsonl/ -type f -name "*.jsonl" | parallel -j 10 "python pdf_classfy_by_type.py --json_file {} >> {/}.jsonl"
-
-```
-
-```shell
-# 如果单独运行脚本，合并到code-clean之后需要运行，参考如下：
-python -m pdf_meta_scan --s3-pdf-path "D:\pdf_files\内容排序测试_pdf\p3_图文混排 5.pdf" --s3-profile s2
-```
-
-## pdf

others/check_inline_formula.py

-# 最终版：把那种text_block有重叠，且inline_formula位置在重叠部分的，认定整个页面都有问题，所有的inline_formula都改成no_check
-from magic_pdf.libs import fitz
-
-
-def check_inline_formula(page, inline_formula_boxes):
-    """
-    :param page :fitz读取的当前页的内容
-    :param inline_formula_boxes: list类型，每一个元素是一个元祖 (L, U, R, D)
-
-    :return: inline_formula_check: list类型，每一个元素是一个类别，其顺序对应输入的inline_formula_boxes，给每个行内公式打一个标签，包括：
-        - nocheck_inline_formula：这个公式框没有与任何span相交，有可能存在问题
-        - wrong_text_block：这个公式框同时存在多个block里，可能页面的text block存在问题
-        - false_inline_formula：只涉及一个span并且只占据这个span的小部分面积，判断可能不是公式
-        - true_inline_formula：两种情况判断为公式，一是横跨多个span，二是只涉及一个span但是几乎占据了这个span大部分的面积
-    """
-
-    # count = defaultdict(int)
-    ## ------------------------ Text --------------------------------------------
-    blocks = page.get_text(
-            "dict",
-            flags=fitz.TEXTFLAGS_TEXT,
-            #clip=clip,
-        )["blocks"]
-    
-    # iterate over the bboxes
-    inline_formula_check = []
-    for result in inline_formula_boxes:
-        (x1, y1, x2, y2) = (result[0], result[1], result[2], result[3])
-        ## 逐个block##
-        in_block = 0
-        for bbox in blocks:
-            # image = cv2.rectangle(image, (int(bbox['bbox'][0]), int(bbox['bbox'][1])), (int(bbox['bbox'][2]), int(bbox['bbox'][3])), (0, 255, 0), 1)
-            if (y1 >= bbox['bbox'][1] and y2 <= bbox['bbox'][3]) and (x1 >= bbox['bbox'][0] and x2 <= bbox['bbox'][2]):       # 判定公式在哪一个block
-                in_block += 1
-                intersect = []
-                # ## 逐个span###
-                for line in bbox['lines']:
-                    if line['bbox'][1] <= ((y2 - y1) / 2) + y1 <= line['bbox'][3]:   # 判断公式在哪一行
-                        for item in line['spans']:
-                            (t_x1, t_y1, t_x2, t_y2) = item['bbox']
-                            if not ((t_x1 < x1 and t_x2 < x1) or (t_x1 > x2 and t_x2 > x2) or (t_y1 < y1 and t_y2 < y1) or (t_y1 > y2 and t_y2 > y2)):   # 判断是否相交
-                                intersect.append(item['bbox'])
-                                # image = cv2.rectangle(image, (int(t_x1), int(t_y1)), (int(t_x2), int(t_y2)), (0, 255, 0), 1)    # 可视化涉及到的span
-
-                # 可视化公式的分类
-                if len(intersect) == 0:  # 没有与任何一个span有相交，这个span或者这个inline_formula_box可能有问题
-                    # print(f'Wrong location, check {img_path}')
-                    inline_formula_check_result = "nocheck_inline_formula"
-                    # count['not_in_line'] += 1
-                elif len(intersect) == 1:  
-                    if abs((intersect[0][2] - intersect[0][0]) - (x2 - x1)) < (x2 - x1)*0.5: # 只涉及一个span但是几乎占据了这个span大部分的面积，判定为公式
-                        # image = cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 1)   
-                        inline_formula_check_result = "true_inline_formula"
-                        # count['one_span_large'] += 1
-                    else:  # 只涉及一个span并且只占据这个span的小部分面积，判断可能不是公式
-                        # image = cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 0, 255), 1)
-                        inline_formula_check_result = "false_inline_formula"
-                        # count['fail'] += 1
-                else:  # 横跨多个span,判定为公式
-                    # image = cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (255, 0, 0), 1)
-                    inline_formula_check_result = "true_inline_formula"
-                    # count['multi_span'] += 1
-                            
-        if in_block == 0:  # 这个公式没有在任何的block里，这个公式可能有问题
-            # image = cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (255, 255, 0), 1)
-            inline_formula_check_result = "nocheck_inline_formula"
-            # count['not_in_block'] += 1
-        elif in_block > 1: # 这个公式存在于多个block里，这个页面可能有问题
-            inline_formula_check_result = "wrong_text_block"
-
-        inline_formula_check.append(inline_formula_check_result)
-
-    return inline_formula_check

others/pdf2text_evaluatePdfLayout.py

-from magic_pdf.libs import fitz             # pyMuPDF库
-
-
-def calculate_overlapRatio_between_rect1_and_rect2(L1: float, U1: float, R1: float, D1: float, L2: float, U2: float, R2: float, D2: float) -> (float, float):
-    # 计算两个rect，重叠面积各占2个rect面积的比例
-    if min(R1, R2) < max(L1, L2) or min(D1, D2) < max(U1, U2):
-        return 0, 0
-    square_1 = (R1 - L1) * (D1 - U1)
-    square_2 = (R2 - L2) * (D2 - U2)
-    if square_1 == 0 or square_2 == 0:
-        return 0, 0
-    square_overlap = (min(R1, R2) - max(L1, L2)) * (min(D1, D2) - max(U1, U2))
-    return square_overlap / square_1, square_overlap / square_2
-
-
-def evaluate_pdf_layout(page_ID: int, page: fitz.Page, json_from_DocXchain_obj: dict):
-    """
-    :param page_ID: int类型，当前page在当前pdf文档中是第page_D页。
-    :param page :fitz读取的当前页的内容
-    :param res_dir_path: str类型，是每一个pdf文档，在当前.py文件的目录下生成一个与pdf文档同名的文件夹，res_dir_path就是文件夹的dir
-    :param json_from_DocXchain_obj: dict类型，把pdf文档送入DocXChain模型中后，提取bbox，结果保存到pdf文档同名文件夹下的 page_ID.json文件中了。json_from_DocXchain_obj就是打开后的dict
-    """
-    DPI = 72  # use this resolution
-    pix = page.get_pixmap(dpi=DPI)
-    pageL = 0
-    pageR = int(pix.w)
-    pageU = 0
-    pageD = int(pix.h)
-
-    #--------- 通过json_from_DocXchain来获取 title ---------#
-    title_bbox_from_DocXChain = []
-
-    xf_json = json_from_DocXchain_obj
-    width_from_json = xf_json['page_info']['width']
-    height_from_json = xf_json['page_info']['height']
-    LR_scaleRatio = width_from_json / (pageR - pageL)
-    UD_scaleRatio = height_from_json / (pageD - pageU)
-
-    # {0: 'title',  # 标题
-    # 1: 'figure', # 图片
-    #  2: 'plain text',  # 文本
-    #  3: 'header',      # 页眉
-    #  4: 'page number', # 页码
-    #  5: 'footnote',    # 脚注
-    #  6: 'footer',      # 页脚
-    #  7: 'table',       # 表格
-    #  8: 'table caption',  # 表格描述
-    #  9: 'figure caption', # 图片描述
-    #  10: 'equation',      # 公式
-    #  11: 'full column',   # 单栏
-    #  12: 'sub column',    # 多栏
-    #  13: 'embedding',     # 嵌入公式
-    #  14: 'isolated'}      # 单行公式
-    LOSS_THRESHOLD = 2000               # 经验值
-    fullColumn_bboxs = []
-    subColumn_bboxs = []
-    plainText_bboxs = []
-    #### read information of plain text
-    for xf in xf_json['layout_dets']:
-        L = xf['poly'][0] / LR_scaleRatio
-        U = xf['poly'][1] / UD_scaleRatio
-        R = xf['poly'][2] / LR_scaleRatio
-        D = xf['poly'][5] / UD_scaleRatio
-        L, R = min(L, R), max(L, R)
-        U, D = min(U, D), max(U, D)
-        if xf['category_id'] == 2:
-            plainText_bboxs.append((L, U, R, D))
-    #### read information of column
-    for xf in xf_json['subfield_dets']:
-        L = xf['poly'][0] / LR_scaleRatio
-        U = xf['poly'][1] / UD_scaleRatio
-        R = xf['poly'][2] / LR_scaleRatio
-        D = xf['poly'][5] / UD_scaleRatio
-        L, R = min(L, R), max(L, R)
-        U, D = min(U, D), max(U, D)
-        if xf['category_id'] == 11:
-            fullColumn_bboxs.append((L, U, R, D))
-        elif xf['category_id'] == 12:
-            subColumn_bboxs.append((L, U, R, D))
-            
-
-    curPage_loss = 0        # 当前页的loss
-    fail_cnt = 0            # Text文本块没被圈到的情形。
-    for L, U, R, D in plainText_bboxs:
-        find = False
-        for L2, U2, R2, D2 in (fullColumn_bboxs + subColumn_bboxs):
-            ratio_1, _ = calculate_overlapRatio_between_rect1_and_rect2(L, U, R, D, L2, U2, R2, D2)
-            if ratio_1 >= 0.9:
-                loss_1 = (L + R) / 2 - (L2 + R2) / 2
-                loss_2 = L - L2
-                cur_loss = min(abs(loss_1), abs(loss_2))
-                curPage_loss += cur_loss
-                find = True
-                break
-        if find == False:
-            fail_cnt += 1
-
-    isSimpleLayout_flag = False
-    if fail_cnt == 0 and len(fullColumn_bboxs) <= 1 and len(subColumn_bboxs) <= 2:
-        if curPage_loss <= LOSS_THRESHOLD:
-            isSimpleLayout_flag  = True
-    
-    return isSimpleLayout_flag, len(fullColumn_bboxs), len(subColumn_bboxs), curPage_loss

others/pdf2text_recogTitle.py

-from magic_pdf.libs.commons import fitz             # pyMuPDF库
-
-
-def parse_titles(page_ID: int, page: fitz.Page, json_from_DocXchain_obj: dict, exclude_bboxes):
-    """
-    :param page_ID: int类型，当前page在当前pdf文档中是第page_D页。
-    :param page :fitz读取的当前页的内容
-    :param res_dir_path: str类型，是每一个pdf文档，在当前.py文件的目录下生成一个与pdf文档同名的文件夹，res_dir_path就是文件夹的dir
-    :param json_from_DocXchain_obj: dict类型，把pdf文档送入DocXChain模型中后，提取bbox，结果保存到pdf文档同名文件夹下的 page_ID.json文件中了。json_from_DocXchain_obj就是打开后的dict
-    """
-    DPI = 72  # use this resolution
-    pix = page.get_pixmap(dpi=DPI)
-    pageL = 0
-    pageR = int(pix.w)
-    pageU = 0
-    pageD = int(pix.h)
-    
-
-    #--------- 通过json_from_DocXchain来获取 title ---------#
-    title_bbox_from_DocXChain = []
-
-    xf_json = json_from_DocXchain_obj
-    width_from_json = xf_json['page_info']['width']
-    height_from_json = xf_json['page_info']['height']
-    LR_scaleRatio = width_from_json / (pageR - pageL)
-    UD_scaleRatio = height_from_json / (pageD - pageU)
-
-    # {0: 'title',  # 标题
-    # 1: 'figure', # 图片
-    #  2: 'plain text',  # 文本
-    #  3: 'header',      # 页眉
-    #  4: 'page number', # 页码
-    #  5: 'footnote',    # 脚注
-    #  6: 'footer',      # 页脚
-    #  7: 'table',       # 表格
-    #  8: 'table caption',  # 表格描述
-    #  9: 'figure caption', # 图片描述
-    #  10: 'equation',      # 公式
-    #  11: 'full column',   # 单栏
-    #  12: 'sub column',    # 多栏
-    #  13: 'embedding',     # 嵌入公式
-    #  14: 'isolated'}      # 单行公式
-    for xf in xf_json['layout_dets']:
-        L = xf['poly'][0] / LR_scaleRatio
-        U = xf['poly'][1] / UD_scaleRatio
-        R = xf['poly'][2] / LR_scaleRatio
-        D = xf['poly'][5] / UD_scaleRatio
-        # L += pageL          # 有的页面，artBox偏移了。不在（0,0）
-        # R += pageL
-        # U += pageU
-        # D += pageU
-        L, R = min(L, R), max(L, R)
-        U, D = min(U, D), max(U, D)
-        if xf['category_id'] == 0 and xf['score'] >= 0.3:
-            title_bbox_from_DocXChain.append((L, U, R, D))
-            
-    
-    title_final_names = []
-    title_final_bboxs = []
-    title_ID = 0
-    for L, U, R, D in title_bbox_from_DocXChain:
-        # cur_title = page.get_pixmap(clip=(L,U,R,D))
-        new_title_name = "title_{}_{}.png".format(page_ID, title_ID)    # 标题name
-        # cur_title.save(res_dir_path + '/' + new_title_name)           # 把标题存储在新建的文件夹，并命名
-        title_final_names.append(new_title_name)                        # 把标题的名字存在list中
-        title_final_bboxs.append((L, U, R, D))
-        title_ID += 1
-        
-
-    title_final_bboxs.sort(key = lambda LURD: (LURD[1], LURD[0]))
-    curPage_all_title_bboxs = title_final_bboxs
-    return curPage_all_title_bboxs
-

others/vali_bbox_sort.py

-import numpy as np
-import tqdm
-import json
-from validation import cal_edit_distance, format_gt_bbox
-from magic_pdf.layout.layout_sort import sort_with_layout
-
-with open('/mnt/petrelfs/share_data/ouyanglinke/OCR/OCR_validation_dataset_final_rotated_formulafix_highdpi_scihub.json', 'r') as f:
-    samples = json.load(f)
-
-
-# labels = []
-# det_res = []
-edit_distance_dict = []
-edit_distance_list = []
-for i, sample in tqdm.tqdm(enumerate(samples)):
-    pdf_name = sample['pdf_name']
-    s3_pdf_path = sample['s3_path']
-    page_num = sample['page']
-    page_width = sample['annotations']['width']
-    page_height = sample['annotations']['height']
-
-    # pre = main(s3_pdf_path, pdf_bin_file_profile, join_path(pdf_model_dir, pdf_name), pdf_model_profile, save_path, page_num)
-    # pre_dict_list = []
-    # for item in pre:
-    #     pre_sample = {
-    #         'box': [item[0],item[1],item[2],item[3]],
-    #         'type': item[7],
-    #         'score': 1
-    #     }
-    #     pre_dict_list.append(pre_sample)
-
-    # det_res.append(pre_dict_list)
-
-    # match_change_dict = {   # 待确认
-    #     "figure": "image",
-    #     "svg_figure": "image",
-    #     "inline_fomula": "equations_inline",
-    #     "fomula": "equation_interline",
-    #     "figure_caption": "text",
-    #     "table_caption": "text",
-    #     "fomula_caption": "text"
-    # }
-    
-    gt_annos = sample['annotations']
-    # matched_label = label_match(gt_annos, match_change_dict)
-    # labels.append(matched_label)
-
-    # 判断排序函数的精度
-    # 目前不考虑caption与图表相同序号的问题
-    ignore_category = ['abandon', 'figure_caption', 'table_caption', 'formula_caption', 'inline_fomula'] 
-    gt_bboxes = format_gt_bbox(gt_annos, ignore_category)
-    sorted_bboxes, _ = sort_with_layout(gt_bboxes, page_width, page_height)
-    if sorted_bboxes:
-        edit_distance = cal_edit_distance(sorted_bboxes)
-        edit_distance_list.append(edit_distance)
-        edit_distance_dict.append({
-            "sample_id": i,
-            "s3_path": s3_pdf_path,
-            "page_num": page_num,
-            "page_s2_path": sample['page_path'],
-            "edit_distance": edit_distance
-        })
-
-
-# label_classes = ["image", "text", "table", "equation_interline"]
-# detect_matrix = detect_val(labels, det_res, label_classes)
-# print('detect_matrix', detect_matrix)
-edit_distance_mean = np.mean(edit_distance_list)
-print('edit_distance_mean', edit_distance_mean)
-
-edit_distance_dict_sorted = sorted(edit_distance_dict, key=lambda x: x['edit_distance'], reverse=True)
-# print(edit_distance_dict_sorted)
-
-result = {
-    "edit_distance_mean": edit_distance_mean,
-    "edit_distance_dict_sorted": edit_distance_dict_sorted
-}
-
-with open('vali_bbox_sort_result.json', 'w') as f:
-    json.dump(result, f)
\ No newline at end of file

others/validation.py

-import numpy as np
-from mmeval import COCODetection
-import distance
-
-
-def reformat_gt_and_pred(labels, det_res, label_classes):
-    preds = []
-    gts = []
-    
-    for idx, (ann, pred) in enumerate(zip(labels, det_res)):
-        # with open(label_path, "r") as f:
-        #     ann = json.load(f)
-        gt_bboxes = []
-        gt_labels = []
-        for item in ann['step_1']['result']:
-            if item['attribute'] in label_classes:
-                gt_bboxes.append([item['x'], item['y'], item['x']+item['width'], item['y']+item['height']])
-                gt_labels.append(label_classes.index(item['attribute']))
-        
-        gts.append({
-            'img_id': idx,
-            'width': ann['width'],
-            'height': ann['height'],
-            'bboxes': np.array(gt_bboxes),
-            'labels': np.array(gt_labels),
-            'ignore_flags': [False]*len(gt_labels),
-        })
-        
-        bboxes = []
-        labels = []
-        scores = []
-        for item in pred:
-            bboxes.append(item['box'])
-            labels.append(label_classes.index(item['type']))
-            scores.append(item['score'])
-        preds.append({
-            'img_id': idx,
-            'bboxes': np.array(bboxes),
-            'scores': np.array(scores),
-            'labels': np.array(labels),
-        })
-    
-    return gts, preds
-
-
-def detect_val(labels, det_res, label_classes):
-    # label_classes = ['inline_formula', "formula"]
-    meta={'CLASSES':tuple(label_classes)}
-    coco_det_metric = COCODetection(dataset_meta=meta, metric=['bbox'])
-
-    gts, preds = reformat_gt_and_pred(labels, det_res, label_classes)
-    
-    res = coco_det_metric(predictions=preds, groundtruths=gts)
-
-    return res
-
-
-def label_match(annotations, match_change_dict):
-    for item in annotations['step_1']['result']:
-        if item['attribute'] in match_change_dict.keys():
-            item['attribute'] = match_change_dict[item['attribute']]
-    return annotations
-
-
-def format_gt_bbox(annotations, ignore_category):
-    gt_bboxes = []
-    for item in annotations['step_1']['result']:
-        if item['textAttribute'] and item['attribute'] not in ignore_category:
-            x0 = item['x']
-            y0 = item['y']
-            x1 = item['x'] + item['width']
-            y1 = item['y'] + item['height']
-            order = item['textAttribute']
-            category = item['attribute']
-            gt_bboxes.append([x0, y0, x1, y1, order, None, None, category])
-    return gt_bboxes
-
-
-def cal_edit_distance(sorted_bboxes):  
-    # order_list = [int(bbox[4]) for bbox in sorted_bboxes]
-    # print(sorted_bboxes[0][0][12])
-    order_list = [int(bbox[12]) for bbox in sorted_bboxes]
-    sorted_order = sorted(order_list, key=int)
-    distance_cal = distance.levenshtein(order_list, sorted_order)
-    if len(order_list) > 0:
-        return distance_cal / len(order_list)
-    else:
-        return 0
-    
\ No newline at end of file

requirements-qa.txt

+pytest
 Levenshtein
 nltk
 rapidfuzz
@@ -12,3 +13,4 @@ scikit-learn
 tqdm
 htmltabletomd
 pypandoc
+pyopenssl==24.0.0
\ No newline at end of file

tests/test_cli/conf/conf.py

@@ -2,7 +2,6 @@ import os
 conf = {
 "code_path": os.environ.get('GITHUB_WORKSPACE'),
 "pdf_dev_path" : os.environ.get('GITHUB_WORKSPACE') + "/tests/test_cli/pdf_dev",
-"pdf_res_path": "/share/quyuan/mineru/data/mineru"
-
+"pdf_res_path": "/tmp"
 }