{"id":13452694,"url":"https://github.com/jprante/elasticsearch-jdbc","last_synced_at":"2025-12-17T14:56:31.671Z","repository":{"id":38435054,"uuid":"4533231","full_name":"jprante/elasticsearch-jdbc","owner":"jprante","description":"JDBC importer for Elasticsearch","archived":false,"fork":false,"pushed_at":"2021-11-05T18:03:03.000Z","size":7823,"stargazers_count":2836,"open_issues_count":416,"forks_count":707,"subscribers_count":228,"default_branch":"master","last_synced_at":"2025-03-23T21:42:16.247Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"","language":"Java","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/jprante.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE.derby","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2012-06-02T23:17:56.000Z","updated_at":"2025-03-23T01:14:09.000Z","dependencies_parsed_at":"2022-08-09T04:15:39.678Z","dependency_job_id":null,"html_url":"https://github.com/jprante/elasticsearch-jdbc","commit_stats":null,"previous_names":[],"tags_count":90,"template":false,"template_full_name":null,"purl":"pkg:github/jprante/elasticsearch-jdbc","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jprante%2Felasticsearch-jdbc","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jprante%2Felasticsearch-jdbc/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jprante%2Felasticsearch-jdbc/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jprante%2Felasticsearch-jdbc/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/jprante","download_url":"https://codeload.github.com/jprante/elasticsearch-jdbc/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jprante%2Felasticsearch-jdbc/sbom","scorecard":{"id":536003,"data":{"date":"2025-08-11","repo":{"name":"github.com/jprante/elasticsearch-jdbc","commit":"10df51a707572ba40abc2009aa02b99b8d28cbf7"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":2.7,"checks":[{"name":"Token-Permissions","score":-1,"reason":"No tokens found","details":null,"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Maintained","score":0,"reason":"0 commit(s) and 0 issue activity found in the last 90 days -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"Code-Review","score":3,"reason":"Found 6/19 approved changesets -- score normalized to 3","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Dangerous-Workflow","score":-1,"reason":"no workflows found","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Binary-Artifacts","score":9,"reason":"binaries present in source code","details":["Warn: binary detected: gradle/wrapper/gradle-wrapper.jar:1"],"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"Vulnerabilities","score":10,"reason":"0 existing vulnerabilities detected","details":null,"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE.txt:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE.txt:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Signed-Releases","score":0,"reason":"Project has not signed or included provenance with any releases.","details":["Warn: release artifact 2.0.0.0 not signed: https://api.github.com/repos/jprante/elasticsearch-jdbc/releases/2037735","Warn: release artifact 2.0.0.0 does not have provenance: https://api.github.com/repos/jprante/elasticsearch-jdbc/releases/2037735"],"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"Branch-Protection","score":0,"reason":"branch protection not enabled on development/release branches","details":["Warn: branch protection not enabled for branch 'master'"],"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"Pinned-Dependencies","score":0,"reason":"dependency not pinned by hash detected -- score normalized to 0","details":["Warn: containerImage not pinned by hash: docker-example/Dockerfile_elasticsearch:1: pin your Docker image by updating elasticsearch:1.7.3 to elasticsearch:1.7.3@sha256:404c72601ec9f5a5c5f517fcda247698da6df861dda5f3b97af79e7e91fb05e5","Warn: containerImage not pinned by hash: docker-example/Dockerfile_elasticsearchjdbc:1","Warn: containerImage not pinned by hash: docker-example/Dockerfile_postgresql:1: pin your Docker image by updating postgres:9.4.5 to postgres:9.4.5@sha256:9cdc73850f7ffca359421ed66b1b888e8baa5ad8ea8e9cfd41b61cf65323a94e","Info:   0 out of   3 containerImage dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 17 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}}]},"last_synced_at":"2025-08-20T06:58:24.010Z","repository_id":38435054,"created_at":"2025-08-20T06:58:24.010Z","updated_at":"2025-08-20T06:58:24.010Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":27784000,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-12-17T02:00:08.291Z","response_time":55,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-07-31T08:00:20.605Z","updated_at":"2025-12-17T14:56:31.653Z","avatar_url":"https://github.com/jprante.png","language":"Java","funding_links":["https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick\u0026hosted_button_id=GVHFQYZ9WZ8HG"],"categories":["Java","Elasticsearch plugins","II. Databases, search engines, big data and machine learning","Elasticsearch","ELK Elastic Logstash Kibana"],"sub_categories":["Integrations and SQL support","4. Client and drivers for databases","Tools"],"readme":"![JDBC](https://github.com/jprante/elasticsearch-jdbc/raw/master/src/site/resources/database-128.png)\n\nImage by [icons8](http://www.iconsdb.com/icons8/?icon=database) Creative Commons Attribution-NoDerivs 3.0 Unported.\n\n# JDBC importer for Elasticsearch\n\n![Travis](https://api.travis-ci.org/jprante/elasticsearch-jdbc.png)\n\nThe Java Database Connection (JDBC) importer allows to fetch data from JDBC sources for\nindexing into [Elasticsearch](http://www.elasticsearch.org).\n\nThe JDBC importer was designed for tabular data. If you have tables with many joins, the JDBC importer\nis limited in the way to reconstruct deeply nested objects to JSON and process object semantics like object identity.\nThough it would be possible to extend the JDBC importer with a mapping feature where all the object properties\ncould be specified, the current solution is focused on rather simple tabular data streams.\n\nAssuming you have a table of name `orders` with a primary key in column `id`, \nyou can issue this from the command line\n\n    bin=$JDBC_IMPORTER_HOME/bin\n    lib=$JDBC_IMPORTER_HOME/lib\n    echo '{\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n            \"url\" : \"jdbc:mysql://localhost:3306/test\",\n            \"user\" : \"\",\n            \"password\" : \"\",\n            \"sql\" : \"select *, id as _id from orders\"\n        }\n    }' | java \\\n           -cp \"${lib}/*\" \\\n           -Dlog4j.configurationFile=${bin}/log4j2.xml \\\n           org.xbib.tools.Runner \\\n           org.xbib.tools.JDBCImporter\n\nAnd that's it. Now you can check your Elasticsearch cluster for the index `jdbc` or your Elasticsearch logs\nabout what happened.\n\n## Compatiblity matrix\n\n| Release date | JDBC Importer version | Elasticsearch version |\n| -------------| ----------------------| ----------------------|\n| Aug 28 2016  | 2.3.4.1               | 2.3.4                 |\n| Aug  1 2016  | 2.3.4.0               | 2.3.4                 |\n| Jul  6 2016  | 2.3.3.1               | 2.3.3                 |\n| May 28 2016  | 2.3.3.0               | 2.3.3                 |\n| May 27 2016  | 2.3.2.0               | 2.3.2                 |\n| Apr  9 2016  | 2.3.1.0               | 2.3.1                 |\n| Apr  9 2016  | 2.2.1.0               | 2.2.1                 |\n| Feb  5 2016  | 2.2.0.0               | 2.2.0                 |\n| Dec 23 2015  | 2.1.1.2               | 2.1.1                 |\n| Nov 29 2015  | 2.1.0.0               | 2.1.0                 |\n| Oct 29 2015  | 2.0.0.1               | 2.0.0                 |\n| Oct 28 2015  | 2.0.0.0               | 2.0.0                 |\n| Oct 23 2015  | 1.7.3.0               | 1.7.3                 |\n| Sep 29 2015  | 1.7.2.1               | 1.7.2                 |\n| Jul 24 2015  | 1.7.0.1               | 1.7.0                 |\n| Jul 24 2015  | 1.6.0.1               | 1.6.0                 |\n| Jun    2015  | 1.5.2.0               | 1.5.2                 |\n\n## Quick links\n\nJDBC importer 2.3.4.0\n\n`http://xbib.org/repository/org/xbib/elasticsearch/importer/elasticsearch-jdbc/2.3.4.0/elasticsearch-jdbc-2.3.4.0-dist.zip`\n\n## Installation\n\n- in the following steps replace `\u003cversion\u003e` by one of the versions above, e.g. `1.7.0.0`\n\n- download the JDBC importer distribution\n\n  `wget http://xbib.org/repository/org/xbib/elasticsearch/importer/elasticsearch-jdbc/\u003cversion\u003e/elasticsearch-jdbc-\u003cversion\u003e-dist.zip`\n\n- unpack     \n    `unzip elasticsearch-jdbc-\u003cversion\u003e-dist.zip`\n\n- go to the unpacked directory (we call it $JDBC_IMPORTER_HOME)\n    `cd elasticsearch-jdbc-\u003cversion\u003e`\n\n- if you do not find the JDBC driver jar in the `lib` directory, download it from your vendor's site \n    and put the driver jar into the `lib` folder\n\n- modify script in the `bin` directory to your needs (Elasticsearch cluster address) \n\n- run script with a command that starts `org.xbib.tools.JDBCImporter` with the `lib` directory on the classpath\n\n## Bundled drivers\n\nThe JDBC importer comes with open source JDBC drivers bundled for your convenience. \nThey are not part of the JDBC importer, hence, there is no support and no guarantee the bundled drivers will work.\nPlease read the JDBC driver license files attached in the distribution.\nJDBC importer does not link against the code of the drivers. If you do not want the drivers jars,\nthey can be safely removed or replaced by other JDBC drivers at your choice.\n\n## Project docs\n\nThe Maven project site is available at [Github](http://jprante.github.io/elasticsearch-jdbc)\n\n## Issues\n\nAll feedback is welcome! If you find issues, please post them at\n[Github](https://github.com/jprante/elasticsearch-jdbc/issues)\n\n# Contact\n\n[Follow me on twitter](https://twitter.com/xbib)\n\nYou find this software useful and want to honor me for my work? Please donate. \nDonations will also help to keep up the development of open source Elasticsearch add-ons.\n\n[![PayPal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick\u0026hosted_button_id=GVHFQYZ9WZ8HG)\n\n# Documentation\n\nThe relational data is internally transformed into structured JSON objects for the schema-less\nindexing model of Elasticsearch documents.\n\nThe importer can fetch data from RDBMS while multithreaded bulk mode ensures high throughput when \nindexing to Elasticsearch.\n\n## JDBC importer definition file\n\nThe general form of a JDBC import specification is a JSON object.\n\n\t{\n\t    \"type\" : \"jdbc\",\n\t    \"jdbc\" : {\n\t         \u003cdefinition\u003e\n\t    }\n\t}\n\t\nExample:\n\t\n\t{\n\t    \"type\" : \"jdbc\",\n\t    \"jdbc\" : {\n            \"url\" : \"jdbc:mysql://localhost:3306/test\",\n            \"user\" : \"\",\n            \"password\" : \"\",\n            \"sql\" : \"select * from orders\",\n            \"index\" : \"myindex\",\n            \"type\" : \"mytype\",\n            ...\t         \n\t    }\n\t}\n\nThe importer can either be executed via stdin (for example with echo)\n\n    bin=$JDBC_IMPORTER_HOME/bin\n    lib=$JDBC_IMPORTER_HOME/lib\n    echo '{\n      ...\n    }' | java \\\n\t\t-cp \"${lib}/*\" \\\n\t\t-Dlog4j.configurationFile=${bin}/log4j2.xml \\\n\t\torg.xbib.tools.Runner \\\n\t\torg.xbib.tools.JDBCImporter\n\nor with explicit file name parameter from command line. Here is an example\nwhere `statefile.json` is a file which is loaded before execution.\n\n\tjava \\\n\t\t-cp \"${lib}/*\" \\\n\t\t-Dlog4j.configurationFile=${bin}/log4j2.xml \\\n\t\torg.xbib.tools.Runner \\\n\t\torg.xbib.tools.JDBCImporter \\\n\t\tstatefile.json\n\nThis style is convenient for subsequent execution controlled by the `statefile` parameter\nif `statefile` is set to `statefile.json`.\n\n### Parameters\n\nHere is the list of parameters for the `jdbc` block in the definition.\n\n`strategy` - the strategy of the JDBC importer, currently implemented: `\"standard\"`, `\"column\"`\n\n`url` - the JDBC driver URL\n\n`user` - the JDBC database user\n\n`password` - the JDBC database password\n\n`sql`  - SQL statement(s), either a string or a list. If a statement ends with .sql, the statement is looked up in the file system. Example for a list of SQL statements:\n\n\t\"sql\" : [\n\t    {\n\t        \"statement\" : \"select ... from ... where a = ?, b = ?, c = ?\",\n\t        \"parameter\" : [ \"value for a\", \"value for b\", \"value for c\" ]\n\t    },\n\t    {\n\t        \"statement\" : \"insert into  ... where a = ?, b = ?, c = ?\",\n\t        \"parameter\" : [ \"value for a\", \"value for b\", \"value for c\" ],\n\t        \"write\" : \"true\"\n\t    },\n\t    {\n\t        \"statement\" : ...\n\t    }\n\t]\n\n`sql.statement` - the SQL statement\n\n`sql.write` - boolean flag, if true, the SQL statement is interpreted as an insert/update statement that needs write access (default: false).\n\n`sql.callable` - boolean flag, if true, the SQL statement is interpreted as a JDBC `CallableStatement` for stored procedures (default: false).\n\n`sql.parameter` - bind parameters for the SQL statement (in order). Some special values can be used with the following meanings:\n\n   * `$now` - the current timestamp\n   * `$state` - the state, one of: BEFORE_FETCH, FETCH, AFTER_FETCH, IDLE, EXCEPTION\n   * `$metrics.counter` - a counter\n   * `$lastrowcount` - number of rows from last statement\n   * `$lastexceptiondate` - SQL timestamp of last exception\n   * `$lastexception` - full stack trace of last exception\n   * `$metrics.lastexecutionstart` - SQL timestamp of the time when last execution started\n   * `$metrics.lastexecutionend` - SQL timestamp of the time when last execution ended\n   * `$metrics.totalrows` - total number of rows fetched\n   * `$metrics.totalbytes` - total number of bytes fetched\n   * `$metrics.failed` - total number of failed SQL executions\n   * `$metrics.succeeded` - total number of succeeded SQL executions\n   \n`locale` - the default locale (used for parsing numerical values, floating point character. Recommended values is \"en_US\")\n\n`timezone` - the timezone for JDBC setTimestamp() calls when binding parameters with timestamp values\n\n`rounding` -  rounding mode for parsing numeric values. Possible values  \"ceiling\", \"down\", \"floor\", \"halfdown\", \"halfeven\", \"halfup\", \"unnecessary\", \"up\"\n\n`scale` -  the precision of parsing numeric values\n\n`autocommit` - `true` if each statement should be automatically executed. Default is `false`\n\n`fetchsize` - the fetchsize for large result sets, most drivers use this to control the amount of rows in the buffer while iterating through the result set\n\n`max_rows` - limit the number of rows fetches by a statement, the rest of the rows is ignored\n\n`max_retries` - the number of retries to (re)connect to a database\n\n`max_retries_wait` - a time value for the time that should be waited between retries. Default is \"30s\"\n\n`resultset_type` - the JDBC result set type, can be TYPE_FORWARD_ONLY, TYPE_SCROLL_SENSITIVE, TYPE_SCROLL_INSENSITIVE. Default is TYPE_FORWARD_ONLY\n\n`resultset_concurrency` - the JDBC result set concurrency, can be CONCUR_READ_ONLY, CONCUR_UPDATABLE. Default is CONCUR_UPDATABLE\n\n`ignore_null_values` - if NULL values should be ignored when constructing JSON documents. Default is `false`\n\n`detect_geo` - if geo polygons / points in SQL columns should be parsed when constructing JSON documents. Default is `true`\n\n`detect_json` - if json structures in SQL columns should be parsed when constructing JSON documents. Default is `true`\n\n`prepare_database_metadata` - if the driver metadata should be prepared as parameters.  Default is `false`\n\n`prepare_resultset_metadata` - if the result set metadata should be prepared as parameters.  Default is `false`\n\n`column_name_map` - a map of aliases that should be used as a replacement for column names of the database. Useful for Oracle 30 char column name limit. Default is `null`\n\n`query_timeout` - a second value for how long an SQL statement is allowed to be executed before it is considered as lost. Default is `1800`\n\n`connection_properties` - a map for the connection properties for driver connection creation. Default is `null`\n\n`schedule` - a single or a list of cron expressions for scheduled execution. Syntax is equivalent to the\nQuartz cron expression format (see below for syntax)\n\n`threadpoolsize` - a thread pool size for the scheduled executions for `schedule` parameter. If set to `1`, all jobs will be executed serially. Default is `4`.\n\n`interval` - a time value for the delay between two runs (default: not set)\n\n`elasticsearch.cluster` - Elasticsearch cluster name\n\n`elasticsearch.host` - array of Elasticsearch host specifications (host name or `host:port`)\n\n`elasticsearch.port` -  port of Elasticsearch host\n\n`elasticsearch.autodiscover` - if `true`, JDBC importer will try to connect to all cluster nodes. Default is `false`\n\n`max_bulk_actions` - the length of each bulk index request submitted (default: 10000)\n\n`max_concurrent_bulk_requests` - the maximum number of concurrent bulk requests (default: 2 * number of CPU cores)\n\n`max_bulk_volume` - a byte size parameter for the maximum volume allowed for a bulk request (default: \"10m\")\n\n`max_request_wait` - a time value for the maximum wait time for a response of a bulk request (default: \"60s\")\n\n`flush_interval` - a time value for the interval period of flushing index docs to a bulk action (default: \"5s\")\n\n`index` - the Elasticsearch index used for indexing\n\n`type` - the Elasticsearch type of the index used for indexing\n\n`index_settings` - optional settings for the Elasticsearch index\n\n`type_mapping` - optional mapping for the Elasticsearch index type\n\n`statefile` - name of a file where the JDBC importer reads or writes state information \n\n`metrics.lastexecutionstart` - the UTC date/time of the begin of the last execution of a single fetch\n\n`metrics.lastexecutionend` - the UTC date/time of the end of the last execution of a single fetch\n\n`metrics.counter` - a counter for metrics, will be incremented after each single fetch\n\n`metrics.enabled` - if `true`, metrics logging is enabled. Default is `false`\n\n`metrics.interval` - the interval between metrics logging. Default is 30 seconds.\n\n`metrics.logger.plain` - if `true`, write metrics log messages in plain text format. Default is `false`\n\n`metrics.logger.json` - if `true`, write metric log messages in JSON format. Default is `false`\n\n## Overview about the default parameter settings\n\n\t{\n\t    \"jdbc\" : {\n\t\t\t\"strategy\" : \"standard\",\n\t        \"url\" : null,\n\t        \"user\" : null,\n\t        \"password\" : null,\n\t        \"sql\" : null,\n\t        \"locale\" : /* equivalent to Locale.getDefault().toLanguageTag() */,\n\t        \"timezone\" : /* equivalent to TimeZone.getDefault() */,\n\t        \"rounding\" : null,\n\t        \"scale\" : 2,\n\t        \"autocommit\" : false,\n\t        \"fetchsize\" : 10, /* if URL contains MySQL JDBC driver URL, this is Integer.MIN */\n\t        \"max_rows\" : 0,\n\t        \"max_retries\" : 3,\n\t        \"max_retries_wait\" : \"30s\",\n\t        \"resultset_type\" : \"TYPE_FORWARD_ONLY\",\n\t        \"resultset_concurreny\" : \"CONCUR_UPDATABLE\",\n\t        \"ignore_null_values\" : false,\n\t        \"prepare_database_metadata\" : false,\n\t        \"prepare_resultset_metadata\" : false,\n\t        \"column_name_map\" : null,\n\t        \"query_timeout\" : 1800,\n\t        \"connection_properties\" : null,\n\t\t\t\"schedule\" : null,\n\t\t\t\"interval\" : 0L,\n\t\t\t\"threadpoolsize\" : 1,\n\t        \"index\" : \"jdbc\",\n\t        \"type\" : \"jdbc\",\n\t        \"index_settings\" : null,\n\t        \"type_mapping\" : null,\n\t\t\t\"max_bulk_actions\" : 10000,\n\t\t\t\"max_concurrent_bulk_requests\" : 2 * available CPU cores,\n\t\t\t\"max_bulk_volume\" : \"10m\",\n\t\t\t\"max_request_wait\" : \"60s\",\n\t\t\t\"flush_interval\" : \"5s\"\n\t    }\n\t}\n\n## Time scheduled execution\n\nSetting a cron expression in the parameter `schedule` enables repeated (or time scheduled) runs.\n\nYou can also define a list of cron expressions (in a JSON array) to schedule for many\ndifferent time schedules.\n\nExample of a `schedule` parameter:\n\n        \"schedule\" : \"0 0-59 0-23 ? * *\"\n\nThis executes JDBC importer every minute, every hour, all the days in the week/month/year.\n\nThe following documentation about the syntax of the cron expression is copied from the Quartz \nscheduler javadoc page.\n\nCron expressions provide the ability to specify complex time combinations such as\n\"At 8:00am every Monday through Friday\" or \"At 1:30am every last Friday of the month\".\n\nCron expressions are comprised of 6 required fields and one optional field separated by\nwhite space. The fields respectively are described as follows:\n\n| Field Name      | Allowed Values      | Allowed Special Characters  |\n| --------------- | ------------------- | ----------------------------|\n| Seconds         | 0-59                | , - * / |\n| Minutes         | 0-59                | , - * / |\n| Hours           | 0-23                | , - * / |\n| Day-of-month    | 1-31                | , - * ? / L W |\n| Month           | 1-12 or JAN-DEC     | , - * / |\n| Day-of-Week     | 1-7 or SUN-SAT      | , - * ? / L # |\n| Year (Optional) | empty, 1970-2199    | , - * / |\n\nThe '*' character is used to specify all values. For example, \"*\" in the minute field means \"every minute\".\n\nThe '?' character is allowed for the day-of-month and day-of-week fields.\nIt is used to specify 'no specific value'.\nThis is useful when you need to specify something in one of the two fields, but not the other.\n\nThe '-' character is used to specify ranges For example \"10-12\" in the hour field means\n\"the hours 10, 11 and 12\".\n\nThe ',' character is used to specify additional values. For example \"MON,WED,FRI\" in the\nday-of-week field means \"the days Monday, Wednesday, and Friday\".\n\nThe '/' character is used to specify increments. For example \"0/15\" in the seconds field means\n\"the seconds 0, 15, 30, and 45\". And \"5/15\" in the seconds field means \"the seconds 5, 20, 35, and 50\".\nSpecifying '*' before the '/' is equivalent to specifying 0 is the value to start with.\nEssentially, for each field in the expression, there is a set of numbers that can be turned on or off.\nFor seconds and minutes, the numbers range from 0 to 59.\nFor hours 0 to 23, for days of the month 0 to 31, and for months 1 to 12.\nThe \"/\" character simply helps you turn on every \"nth\" value in the given set.\nThus \"7/6\" in the month field only turns on month \"7\", it does NOT mean every 6th month,\nplease note that subtlety.\n\nThe 'L' character is allowed for the day-of-month and day-of-week fields.\nThis character is short-hand for \"last\", but it has different meaning in each of the two fields.\nFor example, the value \"L\" in the day-of-month field means \"the last day of the month\" - day\n31 for January, day 28 for February on non-leap years. If used in the day-of-week field by itself,\nit simply means \"7\" or \"SAT\". But if used in the day-of-week field after another value,\nit means \"the last xxx day of the month\" - for example \"6L\" means \"the last friday of the month\".\nYou can also specify an offset from the last day of the month, such as \"L-3\" which would mean\nthe third-to-last day of the calendar month. When using the 'L' option, it is important not\nto specify lists, or ranges of values, as you'll get confusing/unexpected results.\n\nThe 'W' character is allowed for the day-of-month field. This character is used to specify\nthe weekday (Monday-Friday) nearest the given day. As an example, if you were to specify \"15W\"\nas the value for the day-of-month field, the meaning is: \"the nearest weekday to the 15th of the month\".\nSo if the 15th is a Saturday, the trigger will fire on Friday the 14th.\nIf the 15th is a Sunday, the trigger will fire on Monday the 16th.\nIf the 15th is a Tuesday, then it will fire on Tuesday the 15th.\nHowever if you specify \"1W\" as the value for day-of-month, and the 1st is a Saturday,\nthe trigger will fire on Monday the 3rd, as it will not 'jump' over the boundary of a month's days.\nThe 'W' character can only be specified when the day-of-month is a single day,\nnot a range or list of days.\n\nThe 'L' and 'W' characters can also be combined for the day-of-month expression to yield 'LW',\nwhich translates to \"last weekday of the month\".\n\nThe '#' character is allowed for the day-of-week field. This character is used to specify\n\"the nth\" XXX day of the month. For example, the value of \"6#3\" in the day-of-week field means\nthe third Friday of the month (day 6 = Friday and \"#3\" = the 3rd one in the month).\nOther examples: \"2#1\" = the first Monday of the month and \"4#5\" = the fifth Wednesday of the month.\nNote that if you specify \"#5\" and there is not 5 of the given day-of-week in the month,\nthen no firing will occur that month. If the '#' character is used, there can only be\none expression in the day-of-week field (\"3#1,6#3\" is not valid, since there are two expressions).\n\nThe legal characters and the names of months and days of the week are not case sensitive.\n\nNote: Support for specifying both a day-of-week and a day-of-month value is not complete\n(you'll need to use the '?' character in one of these fields).\nOverflowing ranges is supported - that is, having a larger number on the left hand side than the right.\nYou might do 22-2 to catch 10 o'clock at night until 2 o'clock in the morning, or you might have NOV-FEB.\nIt is very important to note that overuse of overflowing ranges creates ranges that don't make sense\nand no effort has been made to determine which interpretation CronExpression chooses.\nAn example would be \"0 0 14-6 ? * FRI-MON\".\n\n## Structured objects\n\nOne of the advantage of SQL queries is the join operation. From many tables, new tuples can be formed.\n\n\t{\n\t    \"type\" : \"jdbc\",\n\t    \"jdbc\" : {\n\t        \"url\" : \"jdbc:mysql://localhost:3306/test\",\n\t        \"user\" : \"\",\n\t        \"password\" : \"\",\n\t        \"sql\" : \"select \\\"relations\\\" as \\\"_index\\\", orders.customer as \\\"_id\\\", orders.customer as \\\"contact.customer\\\", employees.name as \\\"contact.employee\\\" from orders left join employees on employees.department = orders.department order by _id\"\n\t    }\n\t}\n\nFor example, these rows from SQL\n\n\tmysql\u003e select \"relations\" as \"_index\", orders.customer as \"_id\", orders.customer as \"contact.customer\", employees.name as \"contact.employee\"  from orders left join employees on employees.department = orders.department order by _id;\n\t+-----------+-------+------------------+------------------+\n\t| _index    | _id   | contact.customer | contact.employee |\n\t+-----------+-------+------------------+------------------+\n\t| relations | Big   | Big              | Smith            |\n\t| relations | Large | Large            | Müller           |\n\t| relations | Large | Large            | Meier            |\n\t| relations | Large | Large            | Schulze          |\n\t| relations | Huge  | Huge             | Müller           |\n\t| relations | Huge  | Huge             | Meier            |\n\t| relations | Huge  | Huge             | Schulze          |\n\t| relations | Good  | Good             | Müller           |\n\t| relations | Good  | Good             | Meier            |\n\t| relations | Good  | Good             | Schulze          |\n\t| relations | Bad   | Bad              | Jones            |\n\t+-----------+-------+------------------+------------------+\n\t11 rows in set (0.00 sec)\n\nwill generate fewer JSON objects for the index `relations`.\n\n\tindex=relations id=Big {\"contact\":{\"employee\":\"Smith\",\"customer\":\"Big\"}}\n\tindex=relations id=Large {\"contact\":{\"employee\":[\"Müller\",\"Meier\",\"Schulze\"],\"customer\":\"Large\"}}\n\tindex=relations id=Huge {\"contact\":{\"employee\":[\"Müller\",\"Meier\",\"Schulze\"],\"customer\":\"Huge\"}}\n\tindex=relations id=Good {\"contact\":{\"employee\":[\"Müller\",\"Meier\",\"Schulze\"],\"customer\":\"Good\"}}\n\tindex=relations id=Bad {\"contact\":{\"employee\":\"Jones\",\"customer\":\"Bad\"}}\n\nNote how the `employee` column is collapsed into a JSON array. The repeated occurrence of the `_id` column\ncontrols how values are folded into arrays for making use of the Elasticsearch JSON data model. Make sure your SQL query is ordered by `_id`.\n\n\n## Column names for JSON document construction\n\nIn SQL, each column may be labeled. This label is used by the JDBC importer for JSON document\nconstruction. The dot is the path separator for the document strcuture.\n\nFor example\n\n\t{\n\t    \"type\" : \"jdbc\",\n\t    \"jdbc\" : {\n\t        \"url\" : \"jdbc:mysql://localhost:3306/test\",\n\t        \"user\" : \"\",\n\t        \"password\" : \"\",\n\t        \"sql\" : \"select products.name as \\\"product.name\\\", orders.customer as \\\"product.customer.name\\\", orders.quantity * products.price as \\\"product.customer.bill\\\" from products, orders where products.name = orders.product\"\n\t    }\n\t}\n\nthe labeled columns are `product.name`, `product.customer.name`, and `product.customer.bill`.\n\nA data example:\n\n\tmysql\u003e select products.name as \"product.name\", orders.customer as \"product.customer\", orders.quantity * products.price as \"product.customer.bill\" from products, orders where products.name = orders.product ;\n\t+--------------+------------------+-----------------------+\n\t| product.name | product.customer | product.customer.bill |\n\t+--------------+------------------+-----------------------+\n\t| Apples       | Big              |                     1 |\n\t| Bananas      | Large            |                     2 |\n\t| Oranges      | Huge             |                     6 |\n\t| Apples       | Good             |                     2 |\n\t| Oranges      | Bad              |                     9 |\n\t+--------------+------------------+-----------------------+\n\t5 rows in set, 5 warnings (0.00 sec)\n\nThe structured objects constructed from these columns are\n\n\tid=0 {\"product\":{\"name\":\"Apples\",\"customer\":{\"bill\":1.0,\"name\":\"Big\"}}}\n\tid=1 {\"product\":{\"name\":\"Bananas\",\"customer\":{\"bill\":2.0,\"name\":\"Large\"}}}\n\tid=2 {\"product\":{\"name\":\"Oranges\",\"customer\":{\"bill\":6.0,\"name\":\"Huge\"}}}\n\tid=3 {\"product\":{\"name\":\"Apples\",\"customer\":{\"bill\":2.0,\"name\":\"Good\"}}}\n\tid=4 {\"product\":{\"name\":\"Oranges\",\"customer\":{\"bill\":9.0,\"name\":\"Bad\"}}}\n\nThere are column labels with an underscore as prefix that are mapped to special Elasticsearch document parameters for indexing:\n\n\t_index     the index this object should be indexed into\n\t_type      the type this object should be indexed into\n\t_id        the id of this object\n\t_version   the version of this object\n\t_parent    the parent of this object\n\t_ttl       the time-to-live of this object\n\t_routing   the routing of this object\n\nSee also\n\nhttp://www.elasticsearch.org/guide/reference/mapping/parent-field.html\n\nhttp://www.elasticsearch.org/guide/reference/mapping/ttl-field.html\n\nhttp://www.elasticsearch.org/guide/reference/mapping/routing-field.html\n\n\n## Bracket notation for JSON array construction\n\nWhen construction JSON documents, it is often the case you want to group SQL columns into a JSON object and\nline them up into JSON arrays. For allowing this, a bracket notation is used to identify children\nelements that repeat in each child.\n\nNote, because of limitations in identifying SQL column groups, nested document structures may lead to\nrepetitions of the same group. Fortunately, this is harmless to Elasticsearch queries.\n\nExample:\n\n| _id  | blog.name | blog.published      | blog.association[id] | blog.association[name] | blog.attachment[id]   | blog.attachment[name]    |\n| ---- | ----------| --------------------| -------------------- | ---------------------- | --------------------- | ------------------------ |\n| 4679 | Joe       | 2014-01-06 00:00:00 | 3917                 | John                   | 9450                  | /web/q/g/h/57436356.jpg  |\n| 4679 | Joe       | 2014-01-06 00:00:00 | 3917                 | John                   | 9965                  | /web/i/s/q/GS3193626.jpg |\n| 4679 | Joe       | 2014-01-06 00:00:00 | 3917                 | John                   | 9451                  | /web/i/s/q/GS3193626.jpg |\n\nResult:\n\n    {\n        \"blog\" : {\n            \"attachment\": [\n                {\n                    \"name\" : \"/web/q/g/h/57436356.jpg\",\n                    \"id\" : \"9450\"\n                },\n                {\n                    \"name\" : \"/web/i/s/q/GS3193626.jpg\",\n                    \"id\" : \"9965\"\n                },\n                {\n                    \"name\" : \"/web/i/s/q/GS3193626.jpg\",\n                    \"id\" : \"9451\"\n                }\n            ],\n            \"name\" : \"Joe\",\n            \"association\" : [\n                {\n                    \"name\" : \"John\",\n                    \"id\" : \"3917\"\n                },\n                {\n                    \"name\" : \"John\",\n                    \"id\" : \"3917\"\n                },\n                {\n                    \"name\" : \"John\",\n                    \"id\" : \"3917\"\n                }\n             ],\n             \"published\":\"2014-01-06 00:00:00\"\n         }\n    }\n\n## How to fetch a table?\n\nFor fetching a table, a \"select \\*\" (star) query can be used.\nStar queries are the simplest variant of selecting data from a database.\nThey dump tables into Elasticsearch row-by-row. If no `_id` column name is given, IDs will be automatically generated.\n\nFor example\n\n\t{\n\t    \"type\" : \"jdbc\",\n\t    \"jdbc\" : {\n\t        \"url\" : \"jdbc:mysql://localhost:3306/test\",\n\t        \"user\" : \"\",\n\t        \"password\" : \"\",\n\t        \"sql\" : \"select * from orders\"\n\t    }\n\t}\n\nand this table\n\n\tmysql\u003e select * from orders;\n\t+----------+-----------------+---------+----------+---------------------+\n\t| customer | department      | product | quantity | created             |\n\t+----------+-----------------+---------+----------+---------------------+\n\t| Big      | American Fruits | Apples  |        1 | 0000-00-00 00:00:00 |\n\t| Large    | German Fruits   | Bananas |        1 | 0000-00-00 00:00:00 |\n\t| Huge     | German Fruits   | Oranges |        2 | 0000-00-00 00:00:00 |\n\t| Good     | German Fruits   | Apples  |        2 | 2012-06-01 00:00:00 |\n\t| Bad      | English Fruits  | Oranges |        3 | 2012-06-01 00:00:00 |\n\t+----------+-----------------+---------+----------+---------------------+\n\t5 rows in set (0.00 sec)\n\nwill result into the following JSON documents\n\n\tid=\u003crandom\u003e {\"product\":\"Apples\",\"created\":null,\"department\":\"American Fruits\",\"quantity\":1,\"customer\":\"Big\"}\n\tid=\u003crandom\u003e {\"product\":\"Bananas\",\"created\":null,\"department\":\"German Fruits\",\"quantity\":1,\"customer\":\"Large\"}\n\tid=\u003crandom\u003e {\"product\":\"Oranges\",\"created\":null,\"department\":\"German Fruits\",\"quantity\":2,\"customer\":\"Huge\"}\n\tid=\u003crandom\u003e {\"product\":\"Apples\",\"created\":1338501600000,\"department\":\"German Fruits\",\"quantity\":2,\"customer\":\"Good\"}\n\tid=\u003crandom\u003e {\"product\":\"Oranges\",\"created\":1338501600000,\"department\":\"English Fruits\",\"quantity\":3,\"customer\":\"Bad\"}\n\n## How to update a table?\n\nThe JDBC importer allows to write data into the database for maintenance purpose. \n\nWriting back data into the database makes sense for acknowledging fetched data. \n\nExample:\n\n    {\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n\t        \"url\" : \"jdbc:mysql://localhost:3306/test\",\n\t        \"user\" : \"\",\n\t        \"password\" : \"\",\n            \"sql\" : [\n                {\n                    \"statement\" : \"select * from \\\"products\\\"\"\n                },\n                {\n                    \"statement\" : \"delete from \\\"products\\\" where \\\"_job\\\" = ?\",\n                    \"parameter\" : [ \"$job\" ]\n                }\n            ],\n            \"index\" : \"my_jdbc_index\",\n            \"type\" : \"my_jdbc_type\"\n        }\n    }\n\nIn this example, the DB administrator has prepared product rows and attached a `_job` column to it\nto enumerate the product updates incrementally. The assertion is that Elasticsearch should \ndelete all products from the database after they are indexed successfully. The parameter `$job`\nis a counter. The importer state is saved in a file, so the counter is persisted.\n\n## How to select incremental data from a table?\n\nIt is recommended to use timestamps in UTC for synchronization. This example fetches\nall product rows which has added since the last run, using a millisecond resolution\ncolumn `mytimestamp`:\n\n    {\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n\t        \"url\" : \"jdbc:mysql://localhost:3306/test\",\n            \"statefile\" : \"statefile.json\",\n\t        \"user\" : \"\",\n\t        \"password\" : \"\",\n            \"sql\" : [\n                {\n                    \"statement\" : \"select * from products where mytimestamp \u003e ?\",\n                    \"parameter\" : [ \"$metrics.lastexecutionstart\" ]\n                }\n            ],\n            \"index\" : \"my_jdbc_index\",\n            \"type\" : \"my_jdbc_type\"\n        }\n    }\n\nthe first time you run the script, it will generate the statefile.json file like this\n```\n{\n  \"type\" : \"jdbc\",\n  \"jdbc\" : { \n    \"password\" : \"\",\n    \"index\" : \"my_jdbc_index\",\n    \"statefile\" : \"statefile.json\",\n    \"metrics\" : { \n      \"lastexecutionstart\" : \"2016-03-27T06:37:09.165Z\",\n      \"lastexecutionend\" : \"2016-03-27T06:37:09.501Z\",\n      \"counter\" : \"1\" \n    },  \n    \"type\" : \"my_jdbc_type\",\n    \"user\" : \"\",\n    \"url\" : \"jdbc:mysql://localhost:3306/test\",\n    \"sql\" : [ { \n      \"statement\" : \"select * from products where mytimestamp \u003e ?\", \n      \"parameter\" : [ \"$metrics.lastexecutionstart\" ]\n    } ] \n  }\n}\n```\nafter this, you can select incremental data from table.\n\n## Stored procedures or callable statements\n\nStored procedures can also be used for fetchng data, like this example fo MySQL illustrates. \nSee also [Using Stored Procedures](http://docs.oracle.com/javase/tutorial/jdbc/basics/storedprocedures.html)\nfrom where the example is taken.\n\n    create procedure GET_SUPPLIER_OF_COFFEE(\n        IN coffeeName varchar(32), \n        OUT supplierName varchar(40)) \n        begin \n            select SUPPLIERS.SUP_NAME into supplierName \n            from SUPPLIERS, COFFEES \n            where SUPPLIERS.SUP_ID = COFFEES.SUP_ID \n            and coffeeName = COFFEES.COF_NAME; \n            select supplierName; \n        end\n\nNow it is possible to call the procedure from the JDBC importer and index the result in Elasticsearch.\n\n    {\n        \"jdbc\" : {\n            \"url\" : \"jdbc:mysql://localhost:3306/test\",\n            \"user\" : \"\",\n            \"password\" : \"\",\n            \"sql\" : [\n                {\n                    \"callable\" : true,\n                    \"statement\" : \"{call GET_SUPPLIER_OF_COFFEE(?,?)}\",\n                    \"parameter\" : [\n                         \"Colombian\"\n                    ],\n                    \"register\" : {\n                         \"mySupplierName\" : { \"pos\" : 2, \"type\" : \"varchar\" }\n                    }\n                }\n            ],\n            \"index\" : \"my_jdbc_index\",\n            \"type\" : \"my_jdbc_type\"\n        }\n    }\n\nNote, the `parameter` lists the input parameters in the order they should be applied, like in an\nordinary statement. The `register` declares a list of output parameters in the particular order\nthe `pos` number indicates. It is required to declare the JDBC type in the `type` attribute.\n`mySupplierName`, the key of the output parameter, is used as the Elasticsearch field name specification,\nlike the column name specification in an ordinary SQL statement, because column names are not available\nin callable statement result sets.\n\nIf there is more than one result sets returned by a callable statement,\nthe JDBC importer enters a loop and iterates through all result sets.\n\n# How to import from a CSV file?\n\nImporting from a CSV is easy because a CSV JDBC driver is included.\n\nTry something like this\n\n\t{\n\t\t\"type\" : \"jdbc\",\n\t\t\"jdbc\" : {\n\t\t\t\"driver\" : \"org.xbib.jdbc.csv.CsvDriver\",\n\t\t\t\"url\" : \"jdbc:xbib:csv:mydatadir?columnTypes=\u0026separator=,\",\n\t\t\t\"user\" : \"\",\n\t\t\t\"password\" : \"\",\n\t\t\t\"sql\" : \"select * from mycsvfile\"\n\t\t}\n\t}\n\nwhere\n\n`mydatadir` - path to the directory where the CSV file exists\n\n`mycsvfile` - the name of the file\n\n`columnTypes` - column types will be inferred. Default is `String`, where column types will be all set to  string\n\n`separator` - the column separator\n\nFor a full list of the CSV JDBC driver options, see\nhttps://github.com/jprante/jdbc-driver-csv\n\n# Persisted state\n\nThe JDBC importer writes the state after each execution step into a state file which can be set by the\nparameter `statefile`, see above in the parameter documentation. Default setting is not writing \nto state file.\n\nExample:\n\n    \"sql\" : ...,\n    \"statefile\" : \"statefile.json\",\n    ...\n\nYou can use the `statefile` as input for a next JDBC importer invocation, once it is saved. \nThis is useful if you have to restart the JDBC importer. Because the statefile is written\nin prettified JSON, it is also possible to adjust the \nsettings in the statefile if you need to synchronize with the JDBC source.\n\nNote: there must be enough space on disk to write the state file. If disk is full,\nJDBC importer will write zero length files and give error messages in the importer log.\n\n# Monitoring the JDBC importer\n\nMetrics logging can be enabled to watch for the current transfer statistics. \n\nExample:\n\n    \"sql\" : ...,\n    \"schedule\" : ...,\n    \"statefile\" : \"statefile.json\",\n    \"metrics\" : {\n        \"enabled\" : true,\n        \"interval\" : \"1m\",\n        \"logger\" : {\n            \"plain\" : false,\n            \"json\" : true\n        }\n    }\n\nThis configuration enables metrics logging, sets the metrics logging interval to one minute,\nand switches form plain loggin to JSON logging.\n\nIn the `log4j2.xml` configuration file, you can set up how to log. The loggers used for metrics logging are\n\n`metrics.source.plain` - for plain format logging of the source\n\n`metrics.sink.plain` - for plain format logging of the sink\n\n`metrics.source.json` - for JSON format logging of the source\n\n`metrics.sink.json` - for JSON format logging of the sink\n\nSee also the parameter documentation above.\n\n# Developer notes\n\n## Source, Sink, Context\n\nThe JDBC importer consists of three conceptual interfaces than can be implemented separately.\n\nWhen you use the ``strategy`` parameter, the JDBC importer tries to load additional classes before\nfalling back to the ``standard`` strategy.\n\nYou can implement your own strategy by adding your implementation jars to the lib folder and\ndeclaring the implementing classes in the ``META-INF/services`` directory. \n\nSo, it is easy to reuse or replace existing code, or adapt your own JDBC retrieval strategy\nto the unmodified JDBC importer jar.\n\n### Source\n\nThe `Source` models the data producing side. Beside defining the JDBC connect parameters, \nit manages a dual-channel connection to the data producer for reading and for writing.\nThe reading channel is used for fetching data, while the writing channel can update the source.\n\nThe `Source` API can be inspected at \nhttp://jprante.github.io/elasticsearch-jdbc/apidocs/org/xbib/elasticsearch/jdbc/strategy/Source.html\n\n### Sink\n\nThe `Sink` is the abstraction of the destination where all the data is flowing from the source. \nIt controls the resource usage of the bulk indexing method of Elasticsearch. T\nhrottling is possible by limiting the number of bulk actions per request or by the \nmaximum number of concurrent request.\n\nThe `Sink` API can be inspected at \nhttp://jprante.github.io/elasticsearch-jdbc/apidocs/org/xbib/elasticsearch/jdbc/strategy/Sink.html\n\n### Context\n\nThe `Context` is the abstraction to the thread which performs data fetching from the source \nand transports it to the mouth. A 'move' is considered a single step in the execution cycle. \n\nThe `Context` API can be inspected at \nhttp://jprante.github.io/elasticsearch-jdbc/apidocs/org/xbib/elasticsearch/jdbc/strategy/Context.html\n\n## Strategies\n\nThe JDBC importer can be configured for different methods of data transport.\nSuch methods of data transports are called a 'strategy'.\n\nBy default, the JDBC importer implements a ``standard`` strategy.\n\n## Standard strategy\n\nThe standard strategy contains the following steps of processing:\n\n1. fetch data from the JDBC connection\n2. build structured objects and move them to Elasticsearch for indexing or deleting\n\nIn the ``sql`` parameter, a series of SQL statements can be defined which are executed to fetch the data.\n\n## Your custom strategy\n\nIf you want to extend the JDBC importer, for example by your custom password authentication, you could\nextend `org.xbib.elasticsearch.jdbc.strategy.standard.StandardSource`. \nThen, declare your strategy classes in `META-INF/services`. Add your\njar to the classpath and add the `strategy` parameter to the specifications.\n\n# Examples\n\n## PostgreSQL\n\n1. Install PostgreSQL\n\n   Example: PostgreSQL .dmg (Version 9.1.5) for Mac OS X from http://www.enterprisedb.com/products-services-training/pgdownload\n\n   Filename: postgresql-9.1.5-1-osx.dmg\n\n2. Install Elasticsearch\n\n   Follow instructions on https://www.elastic.co/products/elasticsearch\n\n3. Install JDBC importer\n\n    `wget http://xbib.org/repository/org/xbib/elasticsearch/importer/elasticsearch-jdbc/\u003cversion\u003e/elasticsearch-jdbc-\u003cversion\u003e-dist.zip`\n    \n   (update version respectively)\n\n4. Download PostgreSQL JDBC driver\n\n   Check http://jdbc.postgresql.org/download.html\n\n   Current version is JDBC4 Postgresql Driver, Version 9.1-902\n\n   Filname postgresql-9.1-902.jdbc4.jar\n\n5. Copy driver into lib folder\n\n\t    cp postgresql-9.1-902.jdbc4.jar $JDBC_IMPORTER_HOME/lib\n\n6. Start Elasticsearch\n\n7. Start JDBC importer\n\n   This is just a basic example to a database `test` with user `fred` and password `secret`.\n   Use the port configured during PostgreSQL installation. The default is `5432`.\n   ```\n   bin=$JDBC_IMPORTER_HOME/bin\n   lib=$JDBC_IMPORTER_HOME/lib\n   echo '{\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n            \"url\" : \"jdbc:postgresql://localhost:5432/test\",\n            \"user\" : \"fred\",\n            \"password\" : \"secret\",\n            \"sql\" : \"select * from orders\",\n            \"index\" : \"myindex\",\n            \"type\" : \"mytype\"\n        }\n    }' | java \\\n           -cp \"${lib}/*\" \\\n           -Dlog4j.configurationFile=${bin}/log4j2.xml \\\n           org.xbib.tools.Runner \\\n           org.xbib.tools.JDBCImporter\n   ```\n\n8. Check log messages\n\n   In case the user does not exist, Elasticsearch will log a message.\n\n\n## MS SQL Server\n\n1. Download Elasticsearch\n\n2. Install Elasticsearch\n\n   Follow instructions on https://www.elastic.co/products/elasticsearch\n\n3. Install JDBC importer\n\n    `wget http://xbib.org/repository/org/xbib/elasticsearch/importer/elasticsearch-jdbc/\u003cversion\u003e/elasticsearch-jdbc-\u003cversion\u003e-dist.zip`\n    \n   (update version respectively)\n\n4. Download SQL Server JDBC driver from [the vendor](http://msdn.microsoft.com/en-us/sqlserver/aa937724.aspx)\n\n5. Copy driver into lib folder\n\n     cp SQLJDBC4.jar $JDBC_IMPORTER_HOME/lib\n\n6. Set up the database you want to be indexed.\n   [This includes allowing TCP/IP connections](http://stackoverflow.com/questions/2388042/connect-to-sql-server-2008-with-tcp-ip)\n\n7. Start Elasticsearch\n    ```\n    ./elasticsearch.bat\n    ```\n\n8. Start JDBC importer\n    ```\n    bin=$JDBC_IMPORTER_HOME/bin\n    lib=$JDBC_IMPORTER_HOME/lib\n    echo '{\n        \"type\" : \"jdbc\",\n        \"jdbc\": {\n            \"url\":\"jdbc:sqlserver://localhost:1433;databaseName=ICFV\",\n            \"user\":\"elasticsearch\",\n            \"password\":\"elasticsearch\",\n            \"sql\":\"select * from ScoreCards\",\n            \"index\" : \"myindex\",\n            \"type\" : \"mytype\"\n        }\n    }' | java \\\n           -cp \"${lib}/*\" \\\n           -Dlog4j.configurationFile=${bin}/log4j2.xml \\\n           org.xbib.tools.Runner \\\n           org.xbib.tools.JDBCImporter\n    ```\n\n8. You should see messages from the importer in the logfile.\n\n## Index simple geo coordinates from MySQL in Elasticsearch\n\n1. install MySQL e.g. in /usr/local/mysql\n\n2. start MySQL on localhost:3306 (default)\n\n3. prepare a 'test' database in MySQL\n\n4. create empty user '' with empty password '' (this user should exist as default user, otherwise set up a password and adapt the example)\n\n5. execute SQL in \"geo.dump\" /usr/local/mysql/bin/mysql test \u003c src/test/resources/geo.dump\n\n6. then run this script\n   ```\n   curl -XDELETE 'localhost:9200/myjdbc'\n   bin=$JDBC_IMPORTER_HOME/bin\n   lib=$JDBC_IMPORTER_HOME/lib\n   echo '\n   {\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n            \"url\" : \"jdbc:mysql://localhost:3306/test\",\n            \"user\" : \"\",\n            \"password\" : \"\",\n            \"locale\" : \"en_US\",\n            \"sql\" : [\n                {\n                    \"statement\" : \"select \\\"myjdbc\\\" as _index, \\\"mytype\\\" as _type, name as _id, city, zip, address, lat as \\\"location.lat\\\", lon as \\\"location.lon\\\" from geo\"\n                }\n            ],\n            \"index\" : \"myjdbc\",\n            \"type\" : \"mytype\",\n            \"index_settings\" : {\n                \"index\" : {\n                    \"number_of_shards\" : 1\n                }\n            },\n            \"type_mapping\": {\n                \"mytype\" : {\n                    \"properties\" : {\n                        \"location\" : {\n                            \"type\" : \"geo_point\"\n                        }\n                    }\n                }\n            }\n        }\n   }'  | java \\\n                  -cp \"${lib}/*\" \\\n                  -Dlog4j.configurationFile=${bin}/log4j2.xml \\\n                  org.xbib.tools.Runner \\\n                  org.xbib.tools.JDBCImporter\n   echo \"sleeping while importer should run...\"\n   sleep 10\n   curl -XGET 'localhost:9200/myjdbc/_refresh'\n   curl -XPOST 'localhost:9200/myjdbc/_search?pretty' -d '\n   {\n      \"query\": {\n         \"filtered\": {\n           \"query\": {\n              \"match_all\": {\n               }\n           },\n           \"filter\": {\n               \"geo_distance\" : {\n                   \"distance\" : \"20km\",\n                   \"location\" : {\n                        \"lat\" : 51.0,\n                        \"lon\" : 7.0\n                    }\n                }\n            }\n         }\n       }\n   }'\n   ```\n   \n## Index simple geo coordinates from Postgres/PostGIS geometry field in Elasticsearch\n\n1. install Postgres and PostGIS\n\n2. start Postgres on localhost:5432 (default)\n\n3. prepare a 'test' database in Postgres, connect to the database using psql and create the PostGIS extension `CREATE EXTENSION POSTGIS`\n\n4. create user 'test' with password 'test', quit psql\n\n5. import geo table (includes geom field of type geometry) from \"geo.sql\" psql -U test -d test \u003c src/test/resources/geo.sql\n\n6. then run this script. IMPORTANT: note the use of explicit rounding and scale parameter, by default PostGIS will output floats, these will cause you problems in your geom_point in Elasticsearch unless you use specific casts, you have been warned!\n   ```\n   curl -XDELETE 'localhost:9200/myjdbc'\n   bin=$JDBC_IMPORTER_HOME/bin\n   lib=$JDBC_IMPORTER_HOME/lib\n   echo '\n   {\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n            \"url\" : \"jdbc:postgres://localhost:5432/test\",\n            \"user\" : \"test\",\n            \"password\" : \"test\",\n            \"locale\" : \"en_GB\",\n            \"sql\" : \"select geonameid as _id, name, admin1_code, admin2_code, admin3_code, round(ST_Y(geom)::numeric,8) as \\\"location.lat\\\", round(ST_X(geom)::numeric,8) as \\\"location.lon\\\" from geo\",\n            \"index\" : \"myjdbc\",\n            \"type\" : \"mytype\",\n            \"scale\" : 8,\n            \"index_settings\" : {\n                \"index\" : {\n                    \"number_of_shards\" : 1\n                }\n            },\n            \"type_mapping\": {\n                \"mytype\" : {\n                    \"properties\" : {\n                        \"location\" : {\n                            \"type\" : \"geo_point\"\n                        }\n                    }\n                }\n            }\n        }\n   }'  | java \\\n                  -cp \"${lib}/*\" \\\n                  -Dlog4j.configurationFile=${bin}/log4j2.xml \\\n                  org.xbib.tools.Runner \\\n                  org.xbib.tools.JDBCImporter\n   echo \"sleeping while importer should run...\"\n   sleep 10\n   curl -XGET 'localhost:9200/myjdbc/_refresh'\n   curl -XPOST 'localhost:9200/myjdbc/_search?pretty' -d '\n   {\n      \"query\": {\n         \"filtered\": {\n           \"query\": {\n              \"match_all\": {\n               }\n           },\n           \"filter\": {\n               \"geo_distance\" : {\n                   \"distance\" : \"20km\",\n                   \"location\" : {\n                        \"lat\" : 51.477347,\n                        \"lon\" : -0.000850\n                    }\n                }\n            }\n         }\n       }\n   }'\n   ```\n\n\n## Geo shapes\n\nThe JDBC importer understands WKT http://en.wikipedia.org/wiki/Well-known_text \n\"POINT\" and \"POLYGON\" formats and converts them to GeoJSON.\n\nWith MySQL, the `astext` function can format WKT from columns of type `geometry`.\n\nExample:\n\t\n\tmysql -u root test \u003c\u003cEOT\n\tdrop table if exists test.geom;\n\tcreate table test.geom (\n\t\tid integer,\n\t\tg geometry\n\t);\n\tset @g = 'POLYGON((0 0,10 0,10 10,0 10,0 0),(5 5,7 5,7 7,5 7, 5 5))';\n\tinsert into test.geom values (0, GeomFromText(@g));\n\tEOT\n\t\n\tcurl -XDELETE 'localhost:9200/myjdbc'\n\techo '\n\t{\n\t\t\"type\" : \"jdbc\",\n\t\t\"jdbc\" : {\n\t\t\t\"url\" : \"jdbc:mysql://localhost:3306/test\",\n\t\t\t\"user\" : \"\",\n\t\t\t\"password\" : \"\",\n\t\t\t\"locale\" : \"en_US\",\n\t\t\t\"sql\" : \"select \\\"myjdbc\\\" as _index, \\\"mytype\\\" as _type, id as _id, astext(g) as polygon from geom\",\n\t\t\t\"elasticsearch\" : {\n\t\t\t\t \"cluster\" : \"elasticsearch\",\n\t\t\t\t \"host\" : \"localhost\",\n\t\t\t\t \"port\" : 9300\n\t\t\t},\n\t\t\t\"index\" : \"myjdbc\",\n\t\t\t\"type\" : \"mytype\",\n\t\t\t\"index_settings\" : {\n\t\t\t\t\"index\" : {\n\t\t\t\t\t\"number_of_shards\" : 1\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"type_mapping\": {\n\t\t\t\t\"mytype\" : {\n\t\t\t\t\t\"properties\" : {\n\t\t\t\t\t\t\"polygon\" : {\n\t\t\t\t\t\t\t\"type\" : \"geo_shape\",\n\t\t\t\t\t\t\t\"tree\" : \"quadtree\"\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t' | java \\\n\t\t-cp \"${lib}/*\" \\\n\t\t-Dlog4j.configurationFile=${bin}/log4j2.xml \\\n\t\torg.xbib.tools.Runner \\\n\t\torg.xbib.tools.JDBCImporter\n\n## Oracle column name 30 character limit\n\nOracle imposes a 30 character limit on column name aliases. This makes it sometimes hard to define columns names for\nElasticsearch field names. For this, a column name map can be used like this:\n\n    {\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n            \"url\" : \"jdbc:oracle:thin:@//localhost/sid\",\n            \"user\" : \"user\",\n            \"password\" : \"password\",\n            \"sql\" : \"select or_id as \\\"_id\\\", or_tan as \\\"o.t\\\", or_status as \\\"o.s\\\", stages.* from orders, stages where or_id = st_or_id and or_seqno = st_seqno\",\n            \"column_name_map\" : {\n               \"o\" : \"order\",\n               \"t\" : \"transaction_id\",\n               \"s\" : \"status\"\n            }\n        }\n    }\n\n## Connection properties for JDBC driver\n\nFor some JDBC drivers, advanced parameters can be passed that are not specified in the driver URL, \nbut in the JDBC connection properties. You can specifiy connection properties like this:\n\n    {\n        \"type\" : \"jdbc\",\n        \"jdbc\" : {\n            \"url\" : \"jdbc:oracle:thin:@//localhost:1521/sid\",\n            \"user\" : \"user\",\n            \"password\" : \"password\",\n            \"sql\" : \"select ... from ...\",\n            \"connection_properties\" : {\n                \"oracle.jdbc.TcpNoDelay\" : false,\n                \"useFetchSizeWithLongColumn\" : false,\n                \"oracle.net.CONNECT_TIMEOUT\" : 10000,\n                \"oracle.jdbc.ReadTimeout\" : 50000\n            }\n        }\n    }\n\n\n# License\n\nElasticsearch JDBC Importer\n\nCopyright (C) 2012-2015 Jörg Prante\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fjprante%2Felasticsearch-jdbc","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fjprante%2Felasticsearch-jdbc","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fjprante%2Felasticsearch-jdbc/lists"}