Re-create & First Commit

Author: civitaspo

.gitignore

@@ -6,4 +6,8 @@
 /classpath/
 build/
 .idea
+/.settings/
+/.metadata/
+.classpath
+.project
 *.iml

README.md

@@ -10,19 +10,35 @@ This plugin combine rows from file having data format like a table, based on a c
 
 * **on**:
   * **in_column**: name of the column on input. (string, required)
-  * **file_column**: name of the column on file. (string, default is the same as **in_column**)
+  * **file_column**: name of the column on file. (string, required)
 * **file**:
-  * **path**: path of file (string, required)
-  * **format**: file format (string, required, supported: `json`)
-  * **encode**: file encode (string, default is `raw`, supported: `raw`, `gzip`)
+  * **path_prefix**: Path prefix of input files (string, required)
+  * **parser**: Parser configurations (see below [Supported Parser Type](#supported-parser-type)) (hash, required)
+  * **decoders**: Decoder configuration (see below [Supported Decorder Type](#supported-decorder-type)) (array of hash, optional)
+  * **follow_symlinks**: If true, follow symbolic link directories (boolean, default: `false`)
   * **columns**: required columns of data from the file (array of hash, required)
     * **name**: name of the column
-    * **type**: type of the column (see below)
+    * **type**: type of the column (see below [Type of the column](#type-of-the-column))
     * **format**: format of the timestamp if type is timestamp
-    * **timezone**: timezone of the timestamp if type is timestamp  
+    * **timezone**: timezone of the timestamp if type is timestamp
+* **joined_column_prefix**: prefix added to joined column name for prevent duplicating column name (string, default: `"_joined_by_embulk_""`)
 
----
-**type of the column**
+### Supported Parser Type
+
+* You can use all embulk file-parser plugins.
+  * [built-in parser plugins](http://www.embulk.org/docs/built-in.html)
+  * [parser plugins](http://www.embulk.org/plugins/#file-parser).
+* Special Configuration which [embulk-filter-join_file](.) can set in **parser** section.
+  * **columns_option_name**: Set the **file.columns** value to the option which this option indicates. (optional, default: `"columns"`)
+  * **join_file_columns_option_name**: Same as the **columns_option_name** option. Use this if a parser plugin has **columns_option_name** as its owned option.
+
+### Supported Decorder Type
+
+* You can use all embulk file-decorder plugins.
+  * [built-in decorder plugins](http://www.embulk.org/docs/built-in.html)
+  * [decorder plugins](http://www.embulk.org/plugins/#file-decoder)
+
+### Type of the column
 
 |name|description|
 |:---|:---|
@@ -39,47 +55,30 @@ This plugin combine rows from file having data format like a table, based on a c
 filters:
   - type: join_file
     on:
-      in_column: name_id
+      in_column: id
       file_column: id
     file:
-      path: ./master.json
-      format: json
-      encode: raw
+      path_prefix: ./example/json_array_of_hash/*.json
+      parser:
+        type: jsonpath
+        root: "$."
       columns:
         - {name: id, type: long}
         - {name: name, type: string}
+        - {name: created_at, type: timestamp, format: "%Y-%m-%d"}
+        - {name: point, type: double}
+        - {name: time_zone, type: string}
     joined_column_prefix: _joined_by_embulk_
 ```
 
+See [more examples](./example).
+
 ## Run Example
 
 ```
 $ ./gradlew classpath
-$ embulk run -I lib example/config.yml
-```
-
-## Supported Data Format
-* json
-
-### Supported Data Format Example
-
-#### JSON
-
-```
-[
-  {
-    "id": 0,
-    "name": "civitaspo"
-  },
-  {
-    "id": 2,
-    "name": "moriogai"
-  },
-  {
-    "id": 5,
-    "name": "natsume.soseki"
-  }
-]
+$ embulk bundle install --gemfile=example/Gemfile --path vendor/bundle
+$ embulk run -b example -Ilib example/config.yml
 ```
 
 ## Build

build.gradle

@@ -14,12 +14,13 @@ configurations {
 }
 
 version = "0.1.0"
+
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.+"
-    provided "org.embulk:embulk-core:0.8.+"
+    compile  "org.embulk:embulk-core:0.8.29"
+    provided "org.embulk:embulk-core:0.8.29"
     testCompile "junit:junit:4.+"
 }
 
@@ -46,6 +47,7 @@ task checkstyle(type: Checkstyle) {
     classpath = sourceSets.main.output + sourceSets.test.output
     source = sourceSets.main.allJava + sourceSets.test.allJava
 }
+
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
     jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
     script "${project.name}.gemspec"
@@ -57,9 +59,11 @@ task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
     script "pkg/${project.name}-${project.version}.gem"
 }
 
-task "package"(dependsOn: ["gemspec", "classpath"]) << {
-    println "> Build succeeded."
-    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
 }
 
 task gemspec {

example/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'embulk-parser-jsonpath'

example/config.yml renamed to example/json_array_of_hash/config.yml

@@ -16,17 +16,22 @@ in:
 
 filters:
   - type: join_file
-    base_column: {name: id, type: long}
-    counter_column: {name: id, type: long}
+    on:
+      in_column: id
+      file_column: id
+    file:
+      path_prefix: ./example/json_array_of_hash/*.json
+      parser:
+        type: jsonpath
+        root: "$."
+      columns:
+        - {name: id, type: long}
+        - {name: name, type: string}
+        - {name: created_at, type: timestamp, format: "%Y-%m-%d"}
+        - {name: point, type: double}
+        - {name: time_zone, type: string}
     joined_column_prefix: _joined_by_embulk_
-    file_path: example/master.json
-    file_format: json
-    columns:
-      - {name: id, type: long}
-      - {name: name, type: string}
-      - {name: created_at, type: timestamp, format: "%Y-%m-%d"}
-      - {name: point, type: double}
-      - {name: time_zone, type: string}
+
 
 out:
   type: stdout

example/master.json renamed to example/json_array_of_hash/master.json

@@ -1,4 +1,4 @@
-#Wed Jun 21 00:12:09 JST 2017
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

gradle/wrapper/gradle-wrapper.jar

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 
 ##############################################################################
 ##
@@ -154,19 +154,16 @@ if $cygwin ; then
     esac
 fi
 
-# Escape application args
-save ( ) {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
 }
-APP_ARGS=$(save "$@")
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
   cd "$(dirname "$0")"
 fi
 
-exec "$JAVACMD" "$@"
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"