mtfyun/ragflow
1
0 Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-11-10 13:24:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(sdk): align Python SDK with RAGFlow 1.0 dataset API (#21319)

Browse Source
This commit is contained in:
jay77721 authored and GitHub committed 2026-10-22 15:45:08 +08:00
1 parent 5227bb24f6
commit c34e873824
9 files changed
+433 -234
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/release.yml
+6
View File
@@ -261,6 +261,12 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@v5
- name: Run Python SDK unit tests
run: |
uv venv --python 3.13
uv pip install -e sdk/python pytest
.venv/bin/python -m pytest sdk/python/tests -q
- name: Build and push ragflow-sdk
run: |
cd sdk/python && uv build && uv publish --token ${{ secrets.PYPI_API_TOKEN }}
.github/workflows/sep-tests.yml
+19
View File
@@ -204,6 +204,25 @@ jobs:
echo "has_native_deepdoc_changes=${has_native_deepdoc}" >> $GITHUB_OUTPUT
echo "Go: ${has_go}, Web: ${has_web}, Native DeepDoc: ${has_native_deepdoc}"
ragflow_python_sdk_unit:
name: ragflow_python_sdk_unit
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
runs-on: ubuntu-24.04
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v5
- name: Install Python SDK
run: |
uv venv --python 3.13
uv pip install -e sdk/python pytest
- name: Run Python SDK unit tests
run: .venv/bin/python -m pytest sdk/python/tests -q
ragflow_tests_infinity:
name: ragflow_tests_infinity
needs: ragflow_preflight
docs/references/http_api_reference.md
+59 -150
View File
@@ -59,7 +59,7 @@ The following changes require client updates:
| Area | RAGFlow 1.0 Go change | Client action |
|------|------------------------|---------------|
| Dataset parser selection | Dataset create and update use `parser_id`; responses use the canonical value `general` instead of the legacy `chunk_method: "naive"`. | Replace `chunk_method` with `parser_id`. For the general parser, send `"parser_id": "general"`. |
| Dataset parser selection | Dataset create and update use `parse_type` with `parser_id` (BuiltIn) or `pipeline_id` (Pipeline); responses use the canonical value `general` instead of the legacy `chunk_method: "naive"`. | Replace `chunk_method` with `"parse_type": 1` and `"parser_id": "<builtin parser>"`. For the general parser, send `"parser_id": "general"`. |
| Dataset parser configuration | Go stores and returns component-scoped parser settings. Most legacy top-level `parser_config` fields are not mapped into that component structure. | Read the returned component map and send component-scoped keys when changing parser settings. |
| Errors and authentication | Some validation business codes, messages, and authentication HTTP statuses differ from Python. | Branch on documented codes rather than exact message text. |
@@ -537,18 +537,15 @@ Creates a dataset.
- `'Authorization: Bearer <YOUR_API_KEY>'`
- Body:
- `"name"`: `string`
- `"avatar"`: `string`
- `"description"`: `string`
- `"parse_type"`: `int`
- `"parser_id"`: `string`
- `"pipeline_id"`: `string`
- `"language"`: `string`
- `"embedding_model"`: `string`
- `"permission"`: `string`
- `"parser_id"`: `string`
- `"chunk_method"`: `string`
- `"parser_config"`: `object`
- `"parse_type"`: `int`
- `"pipeline_id"`: `string`
##### A request example creating a built-in general dataset
##### A basic request example
```bash
curl --request POST \
@@ -566,7 +563,7 @@ curl --request POST \
##### A request example specifying ingestion pipeline
:::caution WARNING
You must *not* include `"chunk_method"` or `"parser_config"` when specifying an ingestion pipeline.
Do not include `"parser_id"` when specifying an ingestion pipeline.
:::
```bash
@@ -591,14 +588,6 @@ curl --request POST \
- Case-insensitive
If the name is already taken, the dataset is created with a numbered suffix such as `test_1(1)`, `test_1(2)`, and so on.
- `"avatar"`: `string`
Base64 encoding of the avatar.
- Maximum 65535 characters
- `"description"`: `string`
A brief description of the dataset to create.
- Maximum 65535 characters
- `"language"`: `string`
Optional document/dataset language, for example: `"English"` or `"Chinese"`. Leading and trailing whitespace are stripped. After trimming, it must contain at least 1 character and at most 32 characters. The limit counts characters, not UTF-8 bytes. No restricted list of language values is enforced. If omitted, the server/database default is used.
@@ -615,84 +604,40 @@ curl --request POST \
- `"parser_id"`: `enum<string>`
The built-in parser to use. Required when `"parse_type"` is `1`.
- `"general"`: General (default)
- `"book"`: Book
- `"email"`: Email
- `"laws"`: Laws
- `"manual"`: Manual
- `"one"`: One
- `"paper"`: Paper
- `"picture"`: Picture
- `"presentation"`: Presentation
- `"qa"`: Q&A
- `"table"`: Table
- `"tag"`: Tag
- Must not be included when `"parse_type"` is `2`.
- `"chunk_method"`: `enum<string>`
Legacy alias of `"parser_id"` kept for Python-backend compatibility. The Go backend creates datasets with `"parser_id"` instead; do not send `"chunk_method"` together with `"parse_type"` or `"pipeline_id"`.
Available options:
- `"naive"`: General (default)
- `"book"`: Book
- `"email"`: Email
- `"laws"`: Laws
- `"manual"`: Manual
- `"one"`: One
- `"paper"`: Paper
- `"picture"`: Picture
- `"presentation"`: Presentation
- `"qa"`: Q&A
- `"table"`: Table
- `"tag"`: Tag
- `"paper"`: Paper
- `"book"`: Book
- `"laws"`: Laws
- `"presentation"`: Presentation
- `"picture"`: Picture
- `"one"`: One
- `"audio"`: Audio
- `"email"`: Email
- Must be omitted when `"parse_type"` is `2`.
- `"parser_config"`: `object`
The configuration settings for the dataset parser. The attributes in this JSON object vary with the selected `"parser_id"`:
- If `"parser_id"` is `"general"` (`"naive"` alias), the `"parser_config"` object contains the following attributes:
- `"auto_keywords"`: `int`
- Defaults to `0`
- Minimum: `0`
- Maximum: `32`
- `"auto_questions"`: `int`
- Defaults to `0`
- Minimum: `0`
- Maximum: `10`
- `"chunk_token_num"`: `int`
- Defaults to `512`
- Minimum: `1`
- Maximum: `2048`
- `"delimiter"`: `string`
- Defaults to `"\n"`.
- `"html4excel"`: `bool`
- Whether to convert Excel documents into HTML format.
- Defaults to `false`
- `"layout_recognize"`: `string`
- Defaults to `DeepDOC`
- `"tag_kb_ids"`: `array<string>`
- IDs of datasets to be parsed using the ​​Tag chunk method.
- Before setting this, ensure a tag set is created and properly configured. For details, see [Use tag set](https://ragflow.io/docs/dev/dataset_configuration).
- `"task_page_size"`: `int`
- For PDFs only.
- Defaults to `12`
- Minimum: `1`
- `"parent_child"`: `object` Parent-child chunking settings. When enabled, each chunk is further split into smaller child chunks using `children_delimiter`. At retrieval time, matched child chunks are replaced by their parent's full text before being passed to the LLM, giving precise vector matching with broader context.
- `"use_parent_child"`: `bool` Whether to enable parent-child chunking. Defaults to `false`.
- `"children_delimiter"`: `string` The delimiter used to split a parent chunk into child chunks. Only takes effect when `"use_parent_child"` is `true`. Defaults to `"\n"`.
The configuration settings for the dataset parser. RAGFlow 1.0 stores parser settings per ingestion-pipeline component; use component-scoped keys (keys containing `:`) as returned by the API. Legacy flat keys are ignored.
- `"parse_type"`: `int`
How the dataset selects its parser: `1` = built-in (`"parser_id"`), `2` = ingestion pipeline (`"pipeline_id"`).
- Required. Use `1` together with `"parser_id"`, or `2` together with `"pipeline_id"`.
- Do not mix the two options.
- `"parse_type"`: `int`, *Required*
The parser selection mode:
- `1`: BuiltIn. Requires `"parser_id"`.
- `2`: Pipeline. Requires `"pipeline_id"`.
- `"pipeline_id"`: `string`
The ingestion pipeline ID. Can be found in the corresponding URL in the RAGFlow UI.
- Required (along with `"parse_type": 2`) if specifying an ingestion pipeline.
- Required when `"parse_type"` is `2`.
- Must be a 32-character lowercase hexadecimal string, e.g., `"d0bebe30ae2211f0970942010a8e0005"`.
- Must not be included when `"parse_type"` is `1`.
- Must be omitted when `"parse_type"` is `1`.
:::caution WARNING
You can choose either of the following ingestion options when creating a dataset, but *not* both:
`"parse_type"` is required when creating a dataset. Choose exactly one mode:
- Use a built-in parser -- specify `"parse_type": 1` with `"parser_id"` (optionally with `"parser_config"`).
- Use an ingestion pipeline -- specify both `"parse_type": 2` and `"pipeline_id"`.
- `"parse_type": 1` with `"parser_id"` for a builtin parser.
- `"parse_type": 2` with `"pipeline_id"` for an ingestion pipeline.
`"chunk_method"`, `"avatar"`, `"description"`, and `"auto_metadata_config"` are not accepted by the RAGFlow 1.0 create endpoint. Use `"parser_id"` instead of `"chunk_method"`; set avatar and description with the update endpoint, and metadata configuration with the metadata endpoint.
:::
#### Response
@@ -703,13 +648,9 @@ Success:
{
"code": 0,
"data": {
"avatar": null,
"chunk_count": 0,
"chunk_method": "naive",
"create_date": "2025-04-28T18:40:41",
"create_time": 1745836841611,
"created_by": "3af81804241d11f0a6a79f24fc270c7f",
"description": null,
"document_count": 0,
"embedding_model": "BAAI/bge-large-zh-v1.5@BAAI",
"id": "3b4de7d4241d11f0a6a79f24fc270c7f",
@@ -717,20 +658,22 @@ Success:
"name": "RAGFlow example",
"pagerank": 0,
"parser_config": {
"chunk_token_num": 128,
"delimiter": "\\n!?;。；！？",
"html4excel": false,
"layout_recognize": "DeepDOC"
"GeneralChunker:SixApplesFall": {
"chunk_token_size": 128,
"delimiters": ["\n", "!?;。；！？"],
"parent_child": {
"use_parent_child": false,
"children_delimiter": "\n"
}
}
},
"parser_id": "general",
"permission": "me",
"similarity_threshold": 0.2,
"status": "1",
"tenant_id": "3af81804241d11f0a6a79f24fc270c7f",
"token_num": 0,
"update_date": "2025-04-28T18:40:41",
"update_time": 1745836841611,
"keywords_similarity_weight": 0.7,
},
"keywords_similarity_weight": 0.7
}
}
```
@@ -835,8 +778,10 @@ Updates configurations for a specified dataset.
- `"language"`: `string`
- `"embedding_model"`: `string`
- `"permission"`: `string`
- `"chunk_method"`: `string`
- `"pagerank"`: `int`
- `"parse_type"`: `int`
- `"parser_id"`: `string`
- `"pipeline_id"`: `string`
- `"parser_config"`: `object`
##### Request example
@@ -880,49 +825,16 @@ curl --request PUT \
- Default: `0`
- Minimum: `0`
- Maximum: `100`
- `"chunk_method"`: (*Body parameter*), `enum<string>`
The chunking method for the dataset. Available options:
- `"naive"`: General (default)
- `"book"`: Book
- `"email"`: Email
- `"laws"`: Laws
- `"manual"`: Manual
- `"one"`: One
- `"paper"`: Paper
- `"picture"`: Picture
- `"presentation"`: Presentation
- `"qa"`: Q&A
- `"table"`: Table
- `"tag"`: Tag
- `"parse_type"`: (*Body parameter*), `int`
The parser selection mode: `1` for a builtin parser (`parser_id` required) or `2` for an ingestion pipeline (`pipeline_id` required). Required whenever `parser_id` or `pipeline_id` is provided.
- `"parser_id"`: (*Body parameter*), `string`
The builtin parser ID. Required when `parse_type` is `1`; must be omitted when `parse_type` is `2`.
- `"pipeline_id"`: (*Body parameter*), `string`
The ingestion pipeline ID. Required when `parse_type` is `2`; must be omitted when `parse_type` is `1`.
- `"parser_config"`: (*Body parameter*), `object`
The configuration settings for the dataset parser. The attributes in this JSON object vary with the selected `"chunk_method"`:
- If `"chunk_method"` is `"naive"`, the `"parser_config"` object contains the following attributes:
- `"auto_keywords"`: `int`
- Defaults to `0`
- Minimum: `0`
- Maximum: `32`
- `"auto_questions"`: `int`
- Defaults to `0`
- Minimum: `0`
- Maximum: `10`
- `"chunk_token_num"`: `int`
- Defaults to `512`
- Minimum: `1`
- Maximum: `2048`
- `"delimiter"`: `string`
- Defaults to `"\n"`.
- `"html4excel"`: `bool` Indicates whether to convert Excel documents into HTML format.
- Defaults to `false`
- `"layout_recognize"`: `string`
- Defaults to `DeepDOC`
- `"tag_kb_ids"`: `array<string>`. Refer to [Use tag set](https://ragflow.io/docs/dev/dataset_configuration).
- Must include a list of dataset IDs, where each dataset is parsed using the ​​Tag Chunking Method
- `"task_page_size"`: `int` For PDF only.
- Defaults to `12`
- Minimum: `1`
- `"parent_child"`: `object` Parent-child chunking settings. When enabled, each chunk is further split into smaller child chunks using `children_delimiter`. At retrieval time, matched child chunks are replaced by their parent's full text before being passed to the LLM, giving precise vector matching with broader context.
- `"use_parent_child"`: `bool` Whether to enable parent-child chunking. Defaults to `false`.
- `"children_delimiter"`: `string` The delimiter used to split a parent chunk into child chunks. Only takes effect when `"use_parent_child"` is `true`. Defaults to `"\n"`.
The configuration settings for the dataset parser. RAGFlow 1.0 stores parser settings per ingestion-pipeline component; use component-scoped keys (keys containing `:`) as returned by the API. Legacy flat keys are ignored.
`chunk_method` is not supported by RAGFlow 1.0; use `parse_type` together with `parser_id` or `pipeline_id`.
#### Response
@@ -1020,17 +932,12 @@ Success:
"id": "6e211ee0723611efa10a0242ac120007",
"language": "English",
"name": "mysql",
"chunk_method": "naive",
"parser_id": "general",
"parser_config": {
"chunk_token_num": 8192,
"delimiter": "\\n",
"entity_types": [
"organization",
"person",
"location",
"event",
"time"
]
"GeneralChunker:SixApplesFall": {
"chunk_token_size": 8192,
"delimiters": ["\n"]
}
},
"permission": "me",
"similarity_threshold": 0.2,
@@ -1056,7 +963,7 @@ Success (with `include_parsing_status=true`):
"avatar": null,
"cancel_count": 0,
"chunk_count": 30,
"chunk_method": "qa",
"parser_id": "qa",
"create_date": "2026-03-09T18:57:13",
"create_time": 1773053833094,
"created_by": "928f92a210b911f1ac4cc39e0b8fa3ad",
@@ -1069,7 +976,9 @@ Success (with `include_parsing_status=true`):
"language": "English",
"name": "Test Dataset",
"parser_config": {
"llm_id": "deepseek-chat@DeepSeek"
"Extractor:AutoExtractDefault": {
"llm_id": "deepseek-chat@DeepSeek"
}
},
"permission": "me",
"running_count": 0,
docs/references/python_api_reference.md
+46 -66
View File
@@ -149,19 +149,20 @@ When `extra_body.reference_metadata.include` is `true`, each reference chunk may
```python
RAGFlow.create_dataset(
name: str,
avatar: Optional[str] = None,
description: Optional[str] = None,
parse_type: int = 1,
parser_id: str | None = None,
pipeline_id: str | None = None,
embedding_model: Optional[str] = None,
permission: str = "me",
chunk_method: str = "naive",
parser_config: Optional[DataSet.ParserConfig] = None,
auto_metadata_config: Optional[dict[str, Any]] = None
language: Optional[str] = None,
) -> DataSet
```
Creates a dataset.
`avatar`, `description`, `auto_metadata_config`, and `chunk_method` are not accepted by the RAGFlow 1.0 create endpoint. Set avatar and description through `DataSet.update()`, and metadata configuration through `DataSet.update_auto_metadata()`.
#### Parameters
##### name: `string`, *Required*
@@ -171,22 +172,29 @@ The unique name of the dataset to create. It must adhere to the following requir
- Maximum 128 characters.
- Case-insensitive.
##### avatar: `string`
##### parse_type: `int`
Base64 encoding of the avatar. Defaults to `None`
Selects the parser mode. Defaults to `1`.
##### description: `string`
- `1`: BuiltIn. Use the builtin parser selected by `parser_id`.
- `2`: Pipeline. Use the ingestion pipeline selected by `pipeline_id`.
A brief description of the dataset to create. Defaults to `None`.
##### parser_id: `string | None`
The builtin parser ID. Defaults to `"general"` when `parse_type` is `1`.
Available values include `"general"`, `"qa"`, `"manual"`, `"table"`, `"paper"`, `"book"`, `"laws"`, `"presentation"`, `"picture"`, `"one"`, `"audio"`, and `"email"`.
Must be omitted when `parse_type` is `2`.
##### pipeline_id: `string | None`
The ingestion pipeline ID. Required when `parse_type` is `2`; must be omitted when `parse_type` is `1`.
##### embedding_model: `string | None`
The embedding model to use for the dataset. Defaults to `None`.
##### auto_metadata_config: `dict[str, Any] | None`
The automatic metadata extraction configuration for the dataset. Defaults to `None`.
##### permission
Specifies who can access the dataset to create. Available options:
@@ -194,52 +202,19 @@ Specifies who can access the dataset to create. Available options:
- `"me"`: (Default) Only you can manage the dataset.
- `"team"`: All team members can manage the dataset.
##### chunk_method, `string`
##### language: `string | None`
The chunking method of the dataset to create. Available options:
- `"naive"`: General (default)
- `"manual"`: Manual
- `"qa"`: Q&A
- `"table"`: Table
- `"paper"`: Paper
- `"book"`: Book
- `"laws"`: Laws
- `"presentation"`: Presentation
- `"picture"`: Picture
- `"one"`: One
- `"email"`: Email
Optional document/dataset language, for example `"English"` or `"Chinese"`. Defaults to `None`.
##### parser_config: `DataSet.ParserConfig | None`
The parser configuration of the dataset. A `ParserConfig` object's attributes vary based on the selected `chunk_method`:
The parser configuration of the dataset. Defaults to `None`.
- `chunk_method`=`"naive"`:
`{"chunk_token_num":512,"delimiter":"\n","html4excel":False,"layout_recognize":"DeepDOC","raptor":{"use_raptor":False},"parent_child":{"use_parent_child":False,"children_delimiter":"\n"}}`.
- `chunk_method`=`"qa"`:
`{"raptor": {"use_raptor": False}}`
- `chunk_method`=`"manual"`:
`{"raptor": {"use_raptor": False}}`
- `chunk_method`=`"table"`:
`None`
- `chunk_method`=`"paper"`:
`{"raptor": {"use_raptor": False}}`
- `chunk_method`=`"book"`:
`{"raptor": {"use_raptor": False}}`
- `chunk_method`=`"laws"`:
`{"raptor": {"use_raptor": False}}`
- `chunk_method`=`"picture"`:
`None`
- `chunk_method`=`"presentation"`:
`{"raptor": {"use_raptor": False}}`
- `chunk_method`=`"one"`:
`None`
- `chunk_method`=`"email"`:
`None`
RAGFlow 1.0 stores parser settings per ingestion-pipeline component. Use component-scoped keys (keys containing `:`) as returned by the API; legacy flat keys are ignored.
#### Returns
- Success: A `dataset` object.
- Success: A `dataset` object. Its `parser_id`, `pipeline_id`, and derived `parse_type` reflect the selected parser mode.
- Failure: `Exception`
#### Examples
@@ -249,6 +224,12 @@ from ragflow_sdk import RAGFlow
rag_object = RAGFlow(api_key="<YOUR_API_KEY>", base_url="http://<YOUR_BASE_URL>:9380")
dataset = rag_object.create_dataset(name="kb_1")
dataset = rag_object.create_dataset(name="kb_table", parse_type=1, parser_id="table")
dataset = rag_object.create_dataset(
name="kb_pipeline",
parse_type=2,
pipeline_id="<PIPELINE_ID>",
)
```
---
@@ -389,6 +370,11 @@ A dictionary representing the attributes to update, with the following keys:
- `"avatar"`: (*Body parameter*), `string`
The updated base64 encoding of the avatar.
- Maximum 65535 characters
- `"description"`: (*Body parameter*), `string`
The updated dataset description.
- Maximum 65535 characters
- `"language"`: (*Body parameter*), `string`
The updated document/dataset language, for example `"English"` or `"Chinese"`.
- `"embedding_model"`: (*Body parameter*), `string`
The updated embedding model name.
- Ensure that `"chunk_count"` is `0` before updating `"embedding_model"`.
@@ -403,20 +389,14 @@ A dictionary representing the attributes to update, with the following keys:
- Default: `0`
- Minimum: `0`
- Maximum: `100`
- `"chunk_method"`: (*Body parameter*), `enum<string>`
The chunking method for the dataset. Available options:
- `"naive"`: General (default)
- `"book"`: Book
- `"email"`: Email
- `"laws"`: Laws
- `"manual"`: Manual
- `"one"`: One
- `"paper"`: Paper
- `"picture"`: Picture
- `"presentation"`: Presentation
- `"qa"`: Q&A
- `"table"`: Table
- `"tag"`: Tag
- `"parse_type"`: (*Body parameter*), `int`
The parser mode: `1` for a builtin parser or `2` for an ingestion pipeline. Required whenever `parser_id` or `pipeline_id` is provided.
- `"parser_id"`: (*Body parameter*), `string`
The builtin parser ID. Required when `parse_type` is `1`.
- `"pipeline_id"`: (*Body parameter*), `string`
The ingestion pipeline ID. Required when `parse_type` is `2`.
`chunk_method` is not supported by the RAGFlow 1.0 SDK; use `parse_type` together with `parser_id` or `pipeline_id`.
#### Returns
@@ -431,7 +411,7 @@ from ragflow_sdk import RAGFlow
rag_object = RAGFlow(api_key="<YOUR_API_KEY>", base_url="http://<YOUR_BASE_URL>:9380")
dataset = rag_object.list_datasets(name="kb_name")
dataset = dataset[0]
dataset.update({"embedding_model":"BAAI/bge-zh-v1.5", "chunk_method":"manual"})
dataset.update({"embedding_model":"BAAI/bge-zh-v1.5", "parse_type":1, "parser_id":"manual"})
```
---
sdk/python/ragflow_sdk/modules/dataset.py
+15 -2
View File
@@ -34,7 +34,9 @@ class DataSet(Base):
self.permission = "me"
self.document_count = 0
self.chunk_count = 0
self.chunk_method = "naive"
self.parser_id = ""
self.pipeline_id = None
self.language = None
self.parser_config = None
self.pagerank = 0
for k in list(res_dict.keys()):
@@ -42,13 +44,24 @@ class DataSet(Base):
res_dict.pop(k)
super().__init__(rag, res_dict)
@property
def parse_type(self) -> int:
"""1 when parsed by a builtin parser, 2 when parsed by an ingestion pipeline."""
return 2 if self.pipeline_id else 1
def update(self, update_message: dict):
if "chunk_method" in update_message:
raise ValueError("chunk_method is not supported by RAGFlow 1.0; use parse_type with parser_id or pipeline_id")
res = self.put(f"/datasets/{self.id}", update_message)
res = res.json()
if res.get("code") != 0:
raise Exception(res["message"])
self._update_from_dict(self.rag, res.get("data", {}))
data = res.get("data", {})
# BuiltIn datasets omit pipeline_id from the response; clear the previous
# mode before merging so a Pipeline -> BuiltIn update cannot leave stale state.
self.pipeline_id = data.get("pipeline_id") or None
self._update_from_dict(self.rag, data)
return self
def upload_documents(self, document_list: list[dict]):
sdk/python/ragflow_sdk/ragflow.py
+39 -11
View File
@@ -56,26 +56,54 @@ class RAGFlow:
def create_dataset(
self,
name: str,
avatar: Optional[str] = None,
description: Optional[str] = None,
parse_type: int = 1,
parser_id: str | None = None,
pipeline_id: str | None = None,
embedding_model: Optional[str] = None,
permission: str = "me",
chunk_method: str = "naive",
parser_config: Optional[DataSet.ParserConfig] = None,
auto_metadata_config: Optional[dict[str, Any]] = None,
language: Optional[str] = None,
) -> DataSet:
payload = {
"""Create a dataset with a builtin parser or an ingestion pipeline.
``parse_type=1`` selects a builtin parser and defaults ``parser_id`` to
``"general"``. ``parse_type=2`` selects an ingestion pipeline and requires
``pipeline_id``.
"""
if parser_id is not None and not isinstance(parser_id, str):
raise ValueError("parser_id must be a string")
if pipeline_id is not None and not isinstance(pipeline_id, str):
raise ValueError("pipeline_id must be a string")
parser_id = parser_id.strip() if parser_id is not None else None
pipeline_id = pipeline_id.strip() if pipeline_id is not None else None
if parse_type == 1:
if pipeline_id:
raise ValueError("pipeline_id must not be set when parse_type is 1 (BuiltIn)")
if parser_id == "":
raise ValueError("parser_id must not be empty when parse_type is 1 (BuiltIn)")
selection = {"parser_id": parser_id or "general"}
elif parse_type == 2:
if parser_id:
raise ValueError("parser_id must not be set when parse_type is 2 (Pipeline)")
if not pipeline_id:
raise ValueError("pipeline_id is required when parse_type is 2 (Pipeline)")
selection = {"pipeline_id": pipeline_id}
else:
raise ValueError("parse_type must be 1 (BuiltIn) or 2 (Pipeline)")
payload: dict[str, Any] = {
"name": name,
"avatar": avatar,
"description": description,
"embedding_model": embedding_model,
"parse_type": parse_type,
"permission": permission,
"chunk_method": chunk_method,
**selection,
}
if embedding_model is not None:
payload["embedding_model"] = embedding_model
if parser_config is not None:
payload["parser_config"] = parser_config.to_json()
if auto_metadata_config is not None:
payload["auto_metadata_config"] = auto_metadata_config
if language is not None:
payload["language"] = language
res = self.post("/datasets", payload)
res = res.json()
sdk/python/test/test_frontend_api/common.py
+7 -4
View File
@@ -27,25 +27,28 @@ DATASET_NAME_LIMIT = 128
def create_dataset(auth, payload=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}"
res = requests.post(url=url, headers={"Content-Type": "application/json"}, auth=auth, json=payload)
body = {"parse_type": 1, **(payload or {})}
if body["parse_type"] == 1 and body.get("parser_id") is None:
body["parser_id"] = "general"
res = requests.post(url=url, headers={"Content-Type": "application/json", "Authorization": auth}, json=body)
return res.json()
def list_dataset(auth, params=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}"
res = requests.get(url=url, headers={"Content-Type": "application/json"}, auth=auth, params=params)
res = requests.get(url=url, headers={"Content-Type": "application/json", "Authorization": auth}, params=params)
return res.json()
def rm_dataset(auth, dataset_ids):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}"
res = requests.delete(url=url, headers={"Content-Type": "application/json"}, auth=auth, json={"ids": dataset_ids})
res = requests.delete(url=url, headers={"Content-Type": "application/json", "Authorization": auth}, json={"ids": dataset_ids})
return res.json()
def update_dataset(auth, dataset_id, payload=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}"
res = requests.put(url=url, headers={"Content-Type": "application/json"}, auth=auth, json=payload)
res = requests.put(url=url, headers={"Content-Type": "application/json", "Authorization": auth}, json=payload)
return res.json()
sdk/python/test/test_frontend_api/test_dataset.py
+2 -1
View File
@@ -151,7 +151,8 @@ def test_update_different_params_dataset_success(get_auth):
"name": "test_update_dataset",
"description": "test",
"permission": "me",
"chunk_method": "presentation",
"parse_type": 1,
"parser_id": "presentation",
"language": "spanish",
},
)
sdk/python/tests/test_dataset_sdk.py
+240
View File
@@ -0,0 +1,240 @@
import pytest
from ragflow_sdk import RAGFlow
from ragflow_sdk.modules.dataset import DataSet
class FakeResponse:
def __init__(self, payload):
self._payload = payload
def json(self):
return self._payload
class RecordingRAGFlow(RAGFlow):
def __init__(self, response_data=None, update_data=None):
super().__init__(api_key="test-key", base_url="http://localhost:9380")
self.calls = []
self.response_data = response_data
self.update_data = update_data
def post(self, path, json=None, stream=False, files=None):
self.calls.append(("POST", path, json))
return FakeResponse({"code": 0, "data": self.response_data})
def put(self, path, json):
self.calls.append(("PUT", path, json))
return FakeResponse({"code": 0, "data": self.update_data})
def test_create_dataset_defaults_to_builtin_general():
rag = RecordingRAGFlow({"id": "ds-1", "name": "kb", "parser_id": "general"})
dataset = rag.create_dataset(name="kb")
assert rag.calls == [
(
"POST",
"/datasets",
{
"name": "kb",
"parse_type": 1,
"permission": "me",
"parser_id": "general",
},
)
]
assert dataset.parser_id == "general"
assert dataset.pipeline_id is None
assert dataset.parse_type == 1
def test_create_dataset_builtin_payload_uses_parser_fields():
rag = RecordingRAGFlow(
{
"id": "ds-1",
"name": "kb",
"parser_id": "table",
"language": "English",
}
)
parser_config = DataSet.ParserConfig(rag, {"GeneralChunker:SixApplesFall": {"chunk_token_size": 512}})
dataset = rag.create_dataset(
name="kb",
parser_id="table",
embedding_model="BAAI/bge-large-zh-v1.5@BAAI",
parser_config=parser_config,
language="English",
)
assert rag.calls[0][2] == {
"name": "kb",
"parse_type": 1,
"permission": "me",
"parser_id": "table",
"embedding_model": "BAAI/bge-large-zh-v1.5@BAAI",
"parser_config": {"GeneralChunker:SixApplesFall": {"chunk_token_size": 512}},
"language": "English",
}
assert dataset.parser_id == "table"
assert dataset.language == "English"
@pytest.mark.parametrize("pipeline_id", ["", " "])
def test_create_dataset_treats_blank_pipeline_id_as_absent(pipeline_id):
rag = RecordingRAGFlow({"id": "ds-1", "name": "kb", "parser_id": "general"})
rag.create_dataset(name="kb", pipeline_id=pipeline_id)
payload = rag.calls[0][2]
assert payload["parser_id"] == "general"
assert "pipeline_id" not in payload
@pytest.mark.parametrize("parser_id", ["", " "])
def test_create_dataset_treats_blank_parser_id_as_absent_for_pipeline(parser_id):
rag = RecordingRAGFlow(
{
"id": "ds-1",
"name": "kb",
"parser_id": "",
"pipeline_id": "0123456789abcdef0123456789abcdef",
}
)
rag.create_dataset(
name="kb",
parse_type=2,
parser_id=parser_id,
pipeline_id="0123456789abcdef0123456789abcdef",
)
payload = rag.calls[0][2]
assert "parser_id" not in payload
assert payload["pipeline_id"] == "0123456789abcdef0123456789abcdef"
@pytest.mark.parametrize(
("kwargs", "message"),
[
({"parser_id": 123}, "parser_id must be a string"),
({"parse_type": 2, "pipeline_id": 123}, "pipeline_id must be a string"),
],
)
def test_create_dataset_rejects_non_string_parser_ids(kwargs, message):
rag = RecordingRAGFlow({})
# Exercise the runtime guard used when beartype is unavailable.
create_dataset = getattr(RAGFlow.create_dataset, "__wrapped__", RAGFlow.create_dataset)
with pytest.raises(ValueError, match=message):
create_dataset(rag, name="kb", **kwargs)
assert rag.calls == []
@pytest.mark.parametrize("parser_id", ["", " "])
def test_create_dataset_rejects_blank_parser_id_for_builtin(parser_id):
rag = RecordingRAGFlow({})
with pytest.raises(ValueError, match="parser_id must not be empty"):
rag.create_dataset(name="kb", parser_id=parser_id)
assert rag.calls == []
def test_create_dataset_pipeline_payload():
rag = RecordingRAGFlow(
{
"id": "ds-1",
"name": "kb",
"parser_id": "",
"pipeline_id": "0123456789abcdef0123456789abcdef",
}
)
dataset = rag.create_dataset(
name="kb",
parse_type=2,
pipeline_id="0123456789abcdef0123456789abcdef",
permission="team",
)
assert rag.calls[0][2] == {
"name": "kb",
"parse_type": 2,
"permission": "team",
"pipeline_id": "0123456789abcdef0123456789abcdef",
}
assert dataset.pipeline_id == "0123456789abcdef0123456789abcdef"
assert dataset.parse_type == 2
@pytest.mark.parametrize(
("kwargs", "message"),
[
({"parse_type": 1, "pipeline_id": "p1"}, "pipeline_id must not be set"),
({"parse_type": 2, "parser_id": "general", "pipeline_id": "p1"}, "parser_id must not be set"),
({"parse_type": 2}, "pipeline_id is required"),
({"parse_type": 3}, "parse_type must be"),
],
)
def test_create_dataset_rejects_invalid_parser_selection(kwargs, message):
rag = RecordingRAGFlow({})
with pytest.raises(ValueError, match=message):
rag.create_dataset(name="kb", **kwargs)
assert rag.calls == []
def test_dataset_response_keeps_parser_fields():
rag = RecordingRAGFlow({})
dataset = DataSet(
rag,
{
"id": "ds-1",
"name": "kb",
"parser_id": "general",
"pipeline_id": "p1",
"language": "English",
},
)
assert dataset.parser_id == "general"
assert dataset.pipeline_id == "p1"
assert dataset.language == "English"
assert dataset.parse_type == 2
assert not hasattr(dataset, "chunk_method")
def test_update_sends_parser_selection_and_refreshes_mode():
rag = RecordingRAGFlow(update_data={"id": "ds-1", "name": "kb", "parser_id": "table"})
dataset = DataSet(
rag,
{
"id": "ds-1",
"name": "kb",
"parser_id": "general",
"pipeline_id": "p1",
},
)
dataset.update({"parse_type": 1, "parser_id": "table"})
assert rag.calls == [("PUT", "/datasets/ds-1", {"parse_type": 1, "parser_id": "table"})]
assert dataset.parser_id == "table"
assert dataset.pipeline_id is None
assert dataset.parse_type == 1
def test_update_rejects_legacy_chunk_method():
rag = RecordingRAGFlow({})
dataset = DataSet(rag, {"id": "ds-1", "name": "kb", "parser_id": "general"})
with pytest.raises(ValueError, match="chunk_method"):
dataset.update({"chunk_method": "manual"})
assert rag.calls == []