MinerU源码上传

mt
2025年10月31日14:20:47

+# 解决 numpy>=1.21.6 安装错误
+
+## 问题描述
+在安装 MinerU 项目依赖时遇到错误：
+```
+ERROR: Could not find a version that satisfies the requirement numpy>=1.21.6
+```
+
+## 解决方案
+
+### 方案 1：使用 --user 选项安装（推荐）
+
+由于 Windows 权限问题，推荐使用 `--user` 选项安装到用户目录：
+
+```bash
+python -m pip install --user --upgrade pip
+python -m pip install --user numpy>=1.21.6
+```
+
+### 方案 2：使用国内镜像源（推荐中国用户）
+
+如果网络较慢，可以使用国内镜像源加速安装：
+
+#### 临时使用镜像源：
+```bash
+# 使用清华大学镜像源
+python -m pip install --user -i https://pypi.tuna.tsinghua.edu.cn/simple numpy>=1.21.6
+
+# 或者使用阿里云镜像源
+python -m pip install --user -i https://mirrors.aliyun.com/pypi/simple/ numpy>=1.21.6
+
+# 或者使用中科大镜像源
+python -m pip install --user -i https://pypi.mirrors.ustc.edu.cn/simple/ numpy>=1.21.6
+```
+
+#### 永久配置镜像源：
+创建或编辑 pip 配置文件：
+- Windows: `%APPDATA%\pip\pip.ini`
+- 或在用户目录下创建 `pip.ini`
+
+添加以下内容：
+```ini
+[global]
+index-url = https://pypi.tuna.tsinghua.edu.cn/simple
+[install]
+trusted-host = pypi.tuna.tsinghua.edu.cn
+```
+
+### 方案 3：安装完整项目依赖
+
+如果需要安装整个 MinerU 项目，可以：
+
+```bash
+# 使用 --user 选项
+python -m pip install --user -e .
+
+# 或使用国内镜像源
+python -m pip install --user -i https://pypi.tuna.tsinghua.edu.cn/simple -e .
+```
+
+### 方案 4：修复 pip 警告（可选）
+
+如果看到如下警告：
+```
+WARNING: Ignoring invalid distribution ~ip (C:\Python312\Lib\site-packages)
+```
+
+可以尝试修复：
+
+```bash
+# 找到损坏的包目录（通常是 ~ip 或 ~ 开头）
+# 然后手动删除它们，或运行：
+python -m pip cache purge
+```
+
+### 方案 5：以管理员权限运行（如果上述方案都不行）
+
+1. 右键点击命令提示符或 PowerShell
+2. 选择"以管理员身份运行"
+3. 然后执行安装命令
+
+## 验证安装
+
+安装完成后，可以验证：
+
+```bash
+python -c "import numpy; print(numpy.__version__)"
+```
+
+应该输出类似：`2.3.4` 或更高版本
+
+## 注意事项
+
+1. **Python 版本要求**：MinerU 要求 Python >=3.10,<3.14，当前版本 3.12.1 符合要求
+2. **权限问题**：Windows 上安装全局包可能需要管理员权限，推荐使用 `--user` 选项
+3. **网络问题**：如果下载速度慢，强烈建议使用国内镜像源
+4. **PATH 配置**：升级 pip 后，如果提示 PATH 问题，可以手动添加到环境变量，或使用 `python -m pip` 代替 `pip`
+
+## 常用镜像源
+
+- 清华大学：https://pypi.tuna.tsinghua.edu.cn/simple
+- 阿里云：https://mirrors.aliyun.com/pypi/simple/
+- 中科大：https://pypi.mirrors.ustc.edu.cn/simple
+- 豆瓣：https://pypi.douban.com/simple/
+
+

+LICENSE.md
+README.md
+pyproject.toml
+mineru/__init__.py
+mineru/version.py
+mineru.egg-info/PKG-INFO
+mineru.egg-info/SOURCES.txt
+mineru.egg-info/dependency_links.txt
+mineru.egg-info/entry_points.txt
+mineru.egg-info/not-zip-safe
+mineru.egg-info/requires.txt
+mineru.egg-info/top_level.txt
+mineru/backend/__init__.py
+mineru/backend/utils.py
+mineru/backend/customs/__init__.py
+mineru/backend/customs/exporter.py
+mineru/backend/pipeline/__init__.py
+mineru/backend/pipeline/batch_analyze.py
+mineru/backend/pipeline/model_init.py
+mineru/backend/pipeline/model_json_to_middle_json.py
+mineru/backend/pipeline/model_list.py
+mineru/backend/pipeline/para_split.py
+mineru/backend/pipeline/pipeline_analyze.py
+mineru/backend/pipeline/pipeline_magic_model.py
+mineru/backend/pipeline/pipeline_middle_json_mkcontent.py
+mineru/backend/vlm/__init__.py
+mineru/backend/vlm/model_output_to_middle_json.py
+mineru/backend/vlm/utils.py
+mineru/backend/vlm/vlm_analyze.py
+mineru/backend/vlm/vlm_magic_model.py
+mineru/backend/vlm/vlm_middle_json_mkcontent.py
+mineru/cli/__init__.py
+mineru/cli/client.py
+mineru/cli/common.py
+mineru/cli/fast_api.py
+mineru/cli/gradio_app.py
+mineru/cli/models_download.py
+mineru/cli/vlm_vllm_server.py
+mineru/data/__init__.py
+mineru/data/data_reader_writer/__init__.py
+mineru/data/data_reader_writer/base.py
+mineru/data/data_reader_writer/dummy.py
+mineru/data/data_reader_writer/filebase.py
+mineru/data/data_reader_writer/multi_bucket_s3.py
+mineru/data/data_reader_writer/s3.py
+mineru/data/io/__init__.py
+mineru/data/io/base.py
+mineru/data/io/http.py
+mineru/data/io/s3.py
+mineru/data/utils/__init__.py
+mineru/data/utils/exceptions.py
+mineru/data/utils/path_utils.py
+mineru/data/utils/schemas.py
+mineru/model/__init__.py
+mineru/model/layout/__init__.py
+mineru/model/layout/doclayoutyolo.py
+mineru/model/mfd/__init__.py
+mineru/model/mfd/yolo_v8.py
+mineru/model/mfr/__init__.py
+mineru/model/mfr/utils.py
+mineru/model/mfr/pp_formulanet_plus_m/__init__.py
+mineru/model/mfr/pp_formulanet_plus_m/predict_formula.py
+mineru/model/mfr/pp_formulanet_plus_m/processors.py
+mineru/model/mfr/unimernet/Unimernet.py
+mineru/model/mfr/unimernet/__init__.py
+mineru/model/mfr/unimernet/unimernet_hf/__init__.py
+mineru/model/mfr/unimernet/unimernet_hf/modeling_unimernet.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_mbart/__init__.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_mbart/configuration_unimer_mbart.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_mbart/modeling_unimer_mbart.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_mbart/tokenization_unimer_mbart.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_swin/__init__.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_swin/configuration_unimer_swin.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_swin/image_processing_unimer_swin.py
+mineru/model/mfr/unimernet/unimernet_hf/unimer_swin/modeling_unimer_swin.py
+mineru/model/ocr/__init__.py
+mineru/model/ocr/paddleocr2pytorch/__init__.py
+mineru/model/ocr/paddleocr2pytorch/pytorch_paddle.py
+mineru/model/ori_cls/__init__.py
+mineru/model/ori_cls/paddle_ori_cls.py
+mineru/model/reading_order/__init__.py
+mineru/model/reading_order/layout_reader.py
+mineru/model/reading_order/xycut.py
+mineru/model/table/__init__.py
+mineru/model/table/cls/__init__.py
+mineru/model/table/cls/paddle_table_cls.py
+mineru/model/table/rec/RapidTable.py
+mineru/model/table/rec/__init__.py
+mineru/model/table/rec/slanet_plus/__init__.py
+mineru/model/table/rec/slanet_plus/main.py
+mineru/model/table/rec/slanet_plus/matcher.py
+mineru/model/table/rec/slanet_plus/matcher_utils.py
+mineru/model/table/rec/slanet_plus/table_structure.py
+mineru/model/table/rec/slanet_plus/table_structure_utils.py
+mineru/model/table/rec/unet_table/__init__.py
+mineru/model/table/rec/unet_table/main.py
+mineru/model/table/rec/unet_table/table_recover.py
+mineru/model/table/rec/unet_table/table_structure_unet.py
+mineru/model/table/rec/unet_table/utils.py
+mineru/model/table/rec/unet_table/utils_table_line_rec.py
+mineru/model/table/rec/unet_table/utils_table_recover.py
+mineru/model/utils/__init__.py
+mineru/model/utils/pytorchocr/__init__.py
+mineru/model/utils/pytorchocr/base_ocr_v20.py
+mineru/model/utils/pytorchocr/data/__init__.py
+mineru/model/utils/pytorchocr/data/imaug/__init__.py
+mineru/model/utils/pytorchocr/data/imaug/operators.py
+mineru/model/utils/pytorchocr/modeling/__init__.py
+mineru/model/utils/pytorchocr/modeling/common.py
+mineru/model/utils/pytorchocr/modeling/architectures/__init__.py
+mineru/model/utils/pytorchocr/modeling/architectures/base_model.py
+mineru/model/utils/pytorchocr/modeling/backbones/__init__.py
+mineru/model/utils/pytorchocr/modeling/backbones/det_mobilenet_v3.py
+mineru/model/utils/pytorchocr/modeling/backbones/rec_donut_swin.py
+mineru/model/utils/pytorchocr/modeling/backbones/rec_hgnet.py
+mineru/model/utils/pytorchocr/modeling/backbones/rec_lcnetv3.py
+mineru/model/utils/pytorchocr/modeling/backbones/rec_mobilenet_v3.py
+mineru/model/utils/pytorchocr/modeling/backbones/rec_mv1_enhance.py
+mineru/model/utils/pytorchocr/modeling/backbones/rec_pphgnetv2.py
+mineru/model/utils/pytorchocr/modeling/backbones/rec_svtrnet.py
+mineru/model/utils/pytorchocr/modeling/heads/__init__.py
+mineru/model/utils/pytorchocr/modeling/heads/cls_head.py
+mineru/model/utils/pytorchocr/modeling/heads/det_db_head.py
+mineru/model/utils/pytorchocr/modeling/heads/rec_ctc_head.py
+mineru/model/utils/pytorchocr/modeling/heads/rec_multi_head.py
+mineru/model/utils/pytorchocr/modeling/heads/rec_ppformulanet_head.py
+mineru/model/utils/pytorchocr/modeling/heads/rec_unimernet_head.py
+mineru/model/utils/pytorchocr/modeling/necks/__init__.py
+mineru/model/utils/pytorchocr/modeling/necks/db_fpn.py
+mineru/model/utils/pytorchocr/modeling/necks/intracl.py
+mineru/model/utils/pytorchocr/modeling/necks/rnn.py
+mineru/model/utils/pytorchocr/postprocess/__init__.py
+mineru/model/utils/pytorchocr/postprocess/cls_postprocess.py
+mineru/model/utils/pytorchocr/postprocess/db_postprocess.py
+mineru/model/utils/pytorchocr/postprocess/rec_postprocess.py
+mineru/model/utils/pytorchocr/utils/__init__.py
+mineru/model/utils/pytorchocr/utils/resources/arch_config.yaml
+mineru/model/utils/pytorchocr/utils/resources/models_config.yml
+mineru/model/utils/pytorchocr/utils/resources/pp_formulanet_arch_config.yaml
+mineru/model/utils/pytorchocr/utils/resources/dict/ka_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv4_doc_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_arabic_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_cyrillic_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_devanagari_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_el_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_en_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_eslav_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_korean_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_latin_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_ta_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_te_dict.txt
+mineru/model/utils/pytorchocr/utils/resources/dict/ppocrv5_th_dict.txt
+mineru/model/utils/tools/__init__.py
+mineru/model/utils/tools/infer/__init__.py
+mineru/model/utils/tools/infer/predict_cls.py
+mineru/model/utils/tools/infer/predict_det.py
+mineru/model/utils/tools/infer/predict_rec.py
+mineru/model/utils/tools/infer/predict_system.py
+mineru/model/utils/tools/infer/pytorchocr_utility.py
+mineru/model/vlm_vllm_model/__init__.py
+mineru/model/vlm_vllm_model/server.py
+mineru/resources/header.html
+mineru/resources/fasttext-langdetect/lid.176.ftz
+mineru/utils/__init__.py
+mineru/utils/block_pre_proc.py
+mineru/utils/block_sort.py
+mineru/utils/boxbase.py
+mineru/utils/cli_parser.py
+mineru/utils/config_reader.py
+mineru/utils/cut_image.py
+mineru/utils/draw_bbox.py
+mineru/utils/enum_class.py
+mineru/utils/format_utils.py
+mineru/utils/guess_suffix_or_lang.py
+mineru/utils/hash_utils.py
+mineru/utils/language.py
+mineru/utils/llm_aided.py
+mineru/utils/magic_model_utils.py
+mineru/utils/model_utils.py
+mineru/utils/models_download_utils.py
+mineru/utils/ocr_utils.py
+mineru/utils/pdf_classify.py
+mineru/utils/pdf_image_tools.py
+mineru/utils/pdf_reader.py
+mineru/utils/pdf_text_tool.py
+mineru/utils/run_async.py
+mineru/utils/span_block_fix.py
+mineru/utils/span_pre_proc.py
+mineru/utils/table_merge.py
\ No newline at end of file

+

+[console_scripts]
+mineru = mineru.cli:client.main
+mineru-api = mineru.cli.fast_api:main
+mineru-gradio = mineru.cli.gradio_app:main
+mineru-models-download = mineru.cli.models_download:download_models
+mineru-vllm-server = mineru.cli.vlm_vllm_server:main

+

+boto3>=1.28.43
+click>=8.1.7
+loguru>=0.7.2
+numpy>=1.21.6
+pdfminer.six==20250506
+tqdm>=4.67.1
+requests
+httpx
+pillow>=11.0.0
+pypdfium2>=4.30.0
+pypdf>=5.6.0
+reportlab
+pdftext>=0.6.2
+modelscope>=1.26.0
+huggingface-hub>=0.32.4
+json-repair>=0.46.2
+opencv-python>=4.11.0.86
+fast-langdetect<0.3.0,>=0.2.3
+scikit-image<1.0.0,>=0.25.0
+openai<3,>=1.70.0
+beautifulsoup4<5,>=4.13.5
+magika<0.7.0,>=0.6.2
+mineru-vl-utils<1,>=0.1.14
+
+[all]
+mineru[core]
+mineru[vllm]
+
+[api]
+fastapi
+python-multipart
+uvicorn
+
+[core]
+mineru[vlm]
+mineru[pipeline]
+mineru[api]
+mineru[gradio]
+
+[gradio]
+gradio<6,>=5.34
+gradio-pdf>=0.0.22
+
+[pipeline]
+matplotlib<4,>=3.10
+ultralytics<9,>=8.3.48
+doclayout_yolo==0.0.4
+dill<1,>=0.3.8
+PyYAML<7,>=6.0.2
+ftfy<7,>=6.3.1
+shapely<3,>=2.0.7
+pyclipper<2,>=1.3.0
+omegaconf<3,>=2.3.0
+torch<3,>=2.6.0
+torchvision
+transformers!=4.51.0,<5.0.0,>=4.49.0
+onnxruntime>1.17.0
+
+[test]
+mineru[core]
+pytest
+pytest-cov
+coverage
+fuzzywuzzy
+
+[vllm]
+vllm<0.12,>=0.10.1.1
+
+[vlm]
+torch<3,>=2.6.0
+transformers<5.0.0,>=4.51.1
+accelerate>=1.5.1

+mineru

@@ -13,6 +13,7 @@ from ...utils.model_utils import crop_img, get_res_list_from_layout_res, clean_v
 from ...utils.ocr_utils import merge_det_boxes, update_det_boxes, sorted_boxes
 from ...utils.ocr_utils import get_adjusted_mfdetrec_res, get_ocr_result_list, OcrConfidence, get_rotate_crop_image
 from ...utils.pdf_image_tools import get_crop_np_img
+from ...utils.table_cell_bbox import compute_table_cells
 
 YOLO_LAYOUT_BASE_BATCH_SIZE = 1
 MFD_BASE_BATCH_SIZE = 1
@@ -106,6 +107,7 @@ class BatchAnalyze:
                                                 'lang':_lang,
                                                 'table_img':wireless_table_img,
                                                 'wired_table_img':wired_table_img,
+                                                'page_index': index,  # 记录页码索引，用于后续关联
                                               })
 
         # 表格识别 table recognition
@@ -234,6 +236,81 @@ class BatchAnalyze:
                     start_index = html_code.find("<table>")
                     end_index = html_code.rfind("</table>") + len("</table>")
                     table_res_dict["table_res"]["html"] = html_code[start_index:end_index]
+                
+                # 计算表格单元格的bbox和score
+                try:
+                    html_final = table_res_dict["table_res"].get("html", "")
+                    if html_final:
+                        # 获取表格bbox（从poly转换为[x1, y1, x2, y2]）
+                        table_res = table_res_dict.get("table_res", {})
+                        poly = table_res.get("poly", [])
+                        x_coords = None
+                        y_coords = None
+                        
+                        # 安全地检查poly长度和处理numpy数组
+                        try:
+                            poly_len = len(poly) if hasattr(poly, '__len__') else 0
+                            if poly_len >= 8:
+                                # 转换为列表并提取坐标，确保处理numpy数组
+                                poly_list = list(poly) if not isinstance(poly, list) else poly
+                                x_coords = [float(poly_list[0]), float(poly_list[2]), float(poly_list[4]), float(poly_list[6])]
+                                y_coords = [float(poly_list[1]), float(poly_list[3]), float(poly_list[5]), float(poly_list[7])]
+                                table_bbox = [min(x_coords), min(y_coords), max(x_coords), max(y_coords)]
+                            else:
+                                # 如果没有poly，尝试从bbox字段获取
+                                bbox_list = table_res.get("bbox", [])
+                                if hasattr(bbox_list, '__len__') and len(bbox_list) >= 4:
+                                    table_bbox = [float(bbox_list[0]), float(bbox_list[1]), float(bbox_list[2]), float(bbox_list[3])]
+                                else:
+                                    table_bbox = [0, 0, 0, 0]
+                        except (ValueError, TypeError, IndexError) as e:
+                            logger.warning(f"Error processing table bbox from poly: {e}")
+                            # 回退到bbox字段
+                            bbox_list = table_res.get("bbox", [])
+                            if hasattr(bbox_list, '__len__') and len(bbox_list) >= 4:
+                                table_bbox = [float(bbox_list[0]), float(bbox_list[1]), float(bbox_list[2]), float(bbox_list[3])]
+                            else:
+                                table_bbox = [0, 0, 0, 0]
+                        
+                        # 获取score
+                        # 如果score未定义或为0，使用合理的默认值
+                        raw_score = table_res.get("score", 0.8)
+                        span_score = float(raw_score)
+                        
+                        # 如果score为0或未定义，使用默认值
+                        if span_score <= 0:
+                            logger.warning(f"Invalid table score: {span_score}, using default 0.8")
+                            span_score = 0.8
+                        
+                        # 获取OCR结果
+                        ocr_result = table_res_dict.get("ocr_result", None)
+                        
+                        # 计算裁剪偏移量（OCR结果坐标是相对于裁剪后的表格图像的）
+                        crop_info = None
+                        if x_coords is not None and y_coords is not None:
+                            try:
+                                crop_info = {
+                                    'crop_xmin': min(x_coords),
+                                    'crop_ymin': min(y_coords),
+                                }
+                            except Exception as e:
+                                logger.warning(f"Error creating crop_info: {e}")
+                                crop_info = None
+                        
+                        # 计算cells
+                        cells = compute_table_cells(
+                            html_final,
+                            table_bbox,
+                            span_score,
+                            ocr_result,
+                            crop_info=crop_info
+                        )
+                        
+                        # 将cells添加到table_res中（table_res是layout_res中table项的引用，会自动同步）
+                        table_res_dict["table_res"]["cells"] = cells
+                except Exception as e:
+                    logger.warning(f"Failed to compute table cells: {e}")
+                    table_res_dict["table_res"]["cells"] = []
 
         # OCR det
         if self.enable_ocr_det_batch:

@@ -338,6 +338,10 @@ class MagicModel:
                         span['latex'] = latex
                     elif html:
                         span['html'] = html
+                    # 获取单元格级信息（bbox和score）
+                    cells = layout_det.get('cells', None)
+                    if cells:
+                        span['cells'] = cells
                     span['type'] = ContentType.TABLE
                 elif category_id == CategoryId.InlineEquation:
                     span['content'] = layout_det['latex']