[Add][plugin][writer] Add support for Access Database

wgzhao · Nov 2, 2023 · 909f116 · 909f116
1 parent 5422458
commit 909f116
Show file tree

Hide file tree

Showing 11 changed files with 325 additions and 1 deletion.
diff --git a/docs/assets/jobs/accesswriter.json b/docs/assets/jobs/accesswriter.json
@@ -0,0 +1,66 @@
+{
+    "job": {
+        "setting": {
+            "speed": {
+                "channel": 1,
+                "bytes": -1
+            }
+        },
+        "content": [
+            {
+                 "reader": {
+                    "name": "streamreader",
+                    "parameter": {
+                        "column" : [
+                            {
+                                "value": "Addax",
+                                "type": "string"
+                            },
+                            {
+                                "value": 19880808,
+                                "type": "long"
+                            },
+                            {
+                                "value": "1988-08-08 08:08:08",
+                                "type": "date"
+                            },
+                            {
+                                "value": true,
+                                "type": "bool"
+                            },
+                            {
+                                "value": "test",
+                                "type": "bytes"
+                            }
+                        ],
+                        "sliceRecordCount": 1000
+                    }
+                },
+                "writer": {
+                    "name": "accesswriter",
+                    "parameter": {
+                        "username": "wgzhao",
+                        "password": "",
+                        "column": [
+                            "name",
+                            "file_size",
+                            "file_date",
+                            "file_open",
+                            "memo"
+                        ],
+                        "ddl":"create table tbl_test(name varchar(20), file_size int, file_date date, file_open boolean, memo blob);",
+                        "connection": [
+                            {
+                                "jdbcUrl": "jdbc:ucanaccess:////Users/wgzhao/Downloads/AccessThemeDemo.mdb",
+                                "table": [
+                                    "tbl_test"
+                                ]
+                            }
+                        ]
+                    }
+                }
+            }
+        ]
+    }
+}
+
diff --git a/docs/writer/accesswriter.md b/docs/writer/accesswriter.md
@@ -0,0 +1,36 @@
+# Access Writer
+
+AccessWriter 插件实现了写入数据到 [Access][1] 目的表的功能。
+
+## 示例
+
+假定要写入的 Access 表建表语句如下：
+
+```sql
+create table tbl_test(name varchar(20), file_size int, file_date date, file_open boolean, memo blob);
+```
+
+这里使用一份从内存产生到 Access 导入的数据。
+
+=== "job/stream2access.json"
+
+    ```json
+    --8<-- "jobs/accesswriter.json"
+    ```
+
+将上述配置文件保存为 `job/stream2access.json`
+
+### 执行采集命令
+
+执行以下命令进行数据采集
+
+```shell
+bin/addax.sh job/stream2access.json
+```
+
+## 参数说明
+
+因本插件基于[Addax RDBMS Writer][2] 实现，所以参数说明请参考 [Addax RDBMS Writer][2]。
+
+[1]: https://en.wikipedia.org/wiki/Microsoft_Access
+[2]: ../rdbmswriter
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -75,6 +75,7 @@ nav:
       - reader/txtfilereader.md
 
   - 写入插件:
+      - writer/accesswriter.md
       - writer/cassandrawriter.md
       - writer/clickhousewriter.md
       - writer/dbfwriter.md

diff --git a/package.xml b/package.xml
@@ -322,6 +322,14 @@
         </fileSet>
 
         <!-- writer -->
+        <fileSet>
+            <directory>plugin/writer/accesswriter/target/accesswriter-${project.version}/</directory>
+            <includes>
+                <include>**/*.*</include>
+            </includes>
+            <fileMode>0644</fileMode>
+            <outputDirectory>addax-${project.version}</outputDirectory>
+        </fileSet>
         <fileSet>
             <directory>plugin/writer/cassandrawriter/target/cassandrawriter-${project.version}/</directory>
             <includes>

diff --git a/plugin/writer/accesswriter/package.xml b/plugin/writer/accesswriter/package.xml
@@ -0,0 +1,37 @@
+<assembly
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-component-1.1.2.xsd">
+    <id>release</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <includes>
+                <include>*.json</include>
+            </includes>
+            <outputDirectory>plugin/writer/${project.artifactId}</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>target/</directory>
+            <includes>
+                <include>${project.artifactId}-${project.version}.jar</include>
+            </includes>
+            <outputDirectory>plugin/writer/${project.artifactId}</outputDirectory>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>plugin/writer/${project.artifactId}/libs</outputDirectory>
+            <scope>runtime</scope>
+            <excludes>
+                <exclude>com.wgzhao.addax:*</exclude>
+            </excludes>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/plugin/writer/accesswriter/pom.xml b/plugin/writer/accesswriter/pom.xml
@@ -0,0 +1,53 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.wgzhao.addax</groupId>
+        <artifactId>addax-all</artifactId>
+        <version>4.1.3-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+    <artifactId>accesswriter</artifactId>
+    <name>access-writer</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.wgzhao.addax</groupId>
+            <artifactId>addax-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.wgzhao.addax</groupId>
+            <artifactId>addax-rdbms</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>net.sf.ucanaccess</groupId>
+            <artifactId>ucanaccess</artifactId>
+            <version>5.0.1</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <descriptor>package.xml</descriptor>
+                    </descriptors>
+                    <finalName>${project.artifactId}-${project.version}</finalName>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>release</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/.../accesswriter/src/main/java/com/wgzhao/addax/plugin/writer/accesswriter/AccessWriter.java b/.../accesswriter/src/main/java/com/wgzhao/addax/plugin/writer/accesswriter/AccessWriter.java
@@ -0,0 +1,86 @@
+package com.wgzhao.addax.plugin.writer.accesswriter;
+
+
+import com.wgzhao.addax.common.plugin.RecordReceiver;
+import com.wgzhao.addax.common.spi.Writer;
+import com.wgzhao.addax.common.util.Configuration;
+import com.wgzhao.addax.rdbms.util.DataBaseType;
+import com.wgzhao.addax.rdbms.writer.CommonRdbmsWriter;
+
+import java.util.List;
+
+public class AccessWriter extends Writer {
+
+    private static final DataBaseType DATABASE_TYPE = DataBaseType.Access;
+
+    public static class Job extends Writer.Job {
+
+        private Configuration originalConfig = null;
+
+        private CommonRdbmsWriter.Job commonRdbmsWriterJob = null;
+        @Override
+        public void init() {
+            this.originalConfig = getPluginJobConf();
+            this.commonRdbmsWriterJob = new CommonRdbmsWriter.Job(DATABASE_TYPE);
+            this.commonRdbmsWriterJob.init(this.originalConfig);
+        }
+
+        @Override
+        public void preCheck() {
+            this.commonRdbmsWriterJob.writerPreCheck(this.originalConfig, DATABASE_TYPE);
+        }
+        @Override
+        public void prepare() {
+            this.commonRdbmsWriterJob.prepare(this.originalConfig);
+        }
+
+        @Override
+        public void post() {
+            this.commonRdbmsWriterJob.post(this.originalConfig);
+        }
+
+        @Override
+        public void destroy() {
+            this.commonRdbmsWriterJob.destroy(this.originalConfig);
+        }
+
+        @Override
+        public List<Configuration> split(int mandatoryNumber) {
+            return this.commonRdbmsWriterJob.split(this.originalConfig, mandatoryNumber);
+        }
+    }
+
+    public static class Task extends Writer.Task {
+
+        private Configuration configuration;
+
+        private CommonRdbmsWriter.Task commonRdbmsWriterTask;
+
+        @Override
+        public void init() {
+            this.configuration = super.getPluginJobConf();
+            this.commonRdbmsWriterTask = new CommonRdbmsWriter.Task(DATABASE_TYPE);
+            this.commonRdbmsWriterTask.init(this.configuration);
+        }
+
+        @Override
+        public void prepare() {
+            this.commonRdbmsWriterTask.prepare(this.configuration);
+        }
+
+        @Override
+        public void startWrite(RecordReceiver lineReceiver) {
+            this.commonRdbmsWriterTask.startWrite(lineReceiver, this.configuration, super.getTaskPluginCollector());
+        }
+
+        @Override
+        public void post() {
+            this.commonRdbmsWriterTask.post(configuration);
+        }
+
+        @Override
+        public void destroy() {
+            this.commonRdbmsWriterTask.destroy(configuration);
+        }
+    }
+}
diff --git a/plugin/writer/accesswriter/src/main/resources/plugin.json b/plugin/writer/accesswriter/src/main/resources/plugin.json
@@ -0,0 +1,6 @@
+{
+  "name": "accesswriter",
+  "class": "com.wgzhao.addax.plugin.writer.accesswriter.AccessWriter",
+  "description": "writer to access database",
+  "developer": "wgzhao"
+}
diff --git a/plugin/writer/accesswriter/src/main/resources/plugin_job_template.json b/plugin/writer/accesswriter/src/main/resources/plugin_job_template.json
@@ -0,0 +1,22 @@
+{
+  "name": "accesswriter",
+  "parameter": {
+    "writeMode": "insert",
+    "username": "root",
+    "password": "",
+    "column": [
+      "*"
+    ],
+    "preSql": [
+      "delete from @table"
+    ],
+    "connection": [
+      {
+        "jdbcUrl": "jdbc:ucanaccess://<access db file path>",
+        "table": [
+          "addax_tbl"
+        ]
+      }
+    ]
+  }
+}
diff --git a/pom.xml b/pom.xml
@@ -429,6 +429,7 @@
                 <module>plugin/reader/txtfilereader</module>
 
                 <!-- writer -->
+                <module>plugin/writer/accesswriter</module>
                 <module>plugin/writer/cassandrawriter</module>
                 <module>plugin/writer/clickhousewriter</module>
                 <module>plugin/writer/dbfwriter</module>

diff --git a/support_data_sources.md b/support_data_sources.md
@@ -2,7 +2,7 @@
 
 | database/filesystem | read | write | plugin(reader/writer)                   | memo                            |
 |---------------------|------|-------|-----------------------------------------|---------------------------------|
-| Access              | ✓    | x     | accessreader                            | suuport [Access][9]             |
+| Access              | ✓    | ✓     | accessreader/accesswriter               | suuport [Access][9]             |
 | Cassandra           | ✓    | ✓     | cassandrareader/cassandrawriter         |                                 |
 | ClickHouse          | ✓    | ✓     | clickhousereader/clickhousewriter       |                                 |
 | Databend            | ✓    | ✓     | databendreader/databendwriter           | support [Databend][8]           |
@@ -39,11 +39,19 @@
 | TEXT                | ✓    | ✓     | textfilereader/textfilewriter           |                                 |
 
 [1]: https://github.com/Kestrong/datax-elasticsearch
+
 [2]: https://phoenix.apache.org
+
 [3]: https://trino.io
+
 [4]: https://www.taosdata.com/cn/
+
 [5]: http://transwarp.cn/
+
 [6]: https://aws.amazon.com/s3
+
 [7]: https://min.io/
+
 [8]: https://databend.rs
+
 [9]: https://en.wikipedia.org/wiki/Microsoft_Access