backuparcgis-service
Changes
.gitignore 84(+84 -0)
.nvmrc 1(+1 -0)
.vscode/launch.json 18(+18 -0)
index.js 174(+174 -0)
lib/fs-promise.js 31(+31 -0)
lib/https-promise.js 29(+29 -0)
package.json 20(+20 -0)
package-lock.json 43(+43 -0)
readme.md 73(+73 -0)
test/script.js 41(+41 -0)
test/terminal.sh 6(+6 -0)
Details
.gitignore 84(+84 -0)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..edc043b
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,84 @@
+# test output
+test/require/*
+test/terminal/*
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build output
+.nuxt
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
\ No newline at end of file
.nvmrc 1(+1 -0)
diff --git a/.nvmrc b/.nvmrc
new file mode 100644
index 0000000..f9fb144
--- /dev/null
+++ b/.nvmrc
@@ -0,0 +1 @@
+10.15.0
.vscode/launch.json 18(+18 -0)
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000..6347a78
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,18 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "type": "node",
+ "request": "launch",
+ "name": "Launch Program",
+ "program": "${workspaceFolder}/index.js",
+ "args": [
+ "https://services.arcgis.com/uUvqNMGPm7axC2dD/arcgis/rest/services/state_parks/FeatureServer/0",
+ "./test/terminal/"
+ ]
+ }
+ ]
+}
\ No newline at end of file
index.js 174(+174 -0)
diff --git a/index.js b/index.js
new file mode 100644
index 0000000..13270fe
--- /dev/null
+++ b/index.js
@@ -0,0 +1,174 @@
+#!/usr/bin/env node
+
+const axios = require('axios')
+const httpsPromise = require('./lib/https-promise')
+const querystring = require('querystring')
+const fs = require('fs')
+const fsPromises = require('./lib/fs-promise')
+const crypto = require('crypto')
+
+class BackupArcGIS {
+
+ constructor(url, dir) {
+
+ if(!url || !dir) {
+ throw new Error('2 Parameters Required (Service URL, Working Directory Path)')
+ }
+
+ this.serviceUrl = url
+ this.workingDir = dir.slice(-1) == '/' ? dir.slice(0,-1) : dir
+ this.serviceDetails = undefined
+ this.duplicate = false
+
+ }
+
+ // get itemId for file naming
+ async getItemId() {
+
+ try {
+ const response = await axios.get(this.serviceUrl + '?f=json')
+ this.serviceDetails = await response.data
+ return this.serviceDetails.serviceItemId
+ } catch (err) {
+ console.log(err)
+ process.exitCode = 1
+ }
+
+ }
+
+ // stream service geojson to tmp directory
+ async downloadFromService(itemId) {
+
+ const params = {
+ where: '1=1',
+ outSR: '4326',
+ returnGeometry: 'true',
+ f: 'geojson',
+ outFields: '*'
+ }
+
+ try {
+ const url = this.serviceUrl + "/query?" + querystring.stringify(params)
+ const path = `${this.workingDir}/archive/tmp/${itemId}.geojson`
+ const tmpHash = await httpsPromise(url, path)
+ return {
+ tmpPath: path,
+ tmpHash: tmpHash
+ }
+ } catch (err) {
+ console.log(err)
+ process.exitCode = 1
+ }
+
+ }
+
+ async compareLatest(itemId, tmpHash) {
+
+ // get latest file in directory
+ const files = await fsPromises.readdir(`${this.workingDir}/archive/${itemId}`)
+ const latest = files[files.length - 1]
+ const latestHash = await hashFile(`${this.workingDir}/archive/${itemId}/${latest}`, handleErr)
+ if(tmpHash === latestHash) {
+ return true
+ }
+ return false
+
+ }
+
+ async run(url, dir) {
+
+ // check for archive and tmp directory
+ const archiveExists = await fsPromises.exists(`${this.workingDir}/archive`)
+ if(!archiveExists) {
+ await fs.mkdir(`${this.workingDir}/archive`, (err) => {
+ if(!(err && err.code === 'EEXIST')){
+ handleErr(err)
+ }
+ })
+ }
+
+ const tmpExists = await fsPromises.exists(`${this.workingDir}/archive/tmp`)
+ if(!tmpExists) {
+ await fs.mkdir(`${this.workingDir}/archive/tmp`, (err) => {
+ if(!(err && err.code === 'EEXIST')){
+ handleErr(err)
+ }
+ })
+ }
+
+ const itemId = await this.getItemId()
+ const {tmpPath, tmpHash} = await this.downloadFromService(itemId)
+ const pathExists = await fsPromises.exists(`${this.workingDir}/archive/${itemId}`)
+ const outFileName = `${this.workingDir}/archive/${itemId}/${Date.now()}.geojson`
+
+ if(pathExists) {
+ // path already exists, check file hashes
+ const exists = await this.compareLatest(itemId, tmpHash)
+ if(!exists) {
+ await fs.rename(tmpPath, outFileName, handleErr)
+ } else {
+ // no updates to file, duplicate
+ this.duplicate = true
+ await fs.unlink(tmpPath, handleErr)
+ }
+ } else {
+ // async version is causing issues
+ fs.mkdirSync(`${this.workingDir}/archive/${itemId}`, handleErr)
+ await fs.rename(tmpPath, outFileName, handleErr)
+ }
+
+ return {
+ filename: this.duplicate ? false : outFileName,
+ serviceDetails: this.serviceDetails,
+ duplicate: this.duplicate
+ }
+ }
+}
+
+function handleErr(err) {
+
+ if(err) throw err
+
+}
+
+function hashFile(path) {
+
+ return new Promise(function (resolve, reject) {
+ const hash = crypto.createHash('sha256')
+ const input = fs.createReadStream(path)
+ input.on('readable', () => {
+ const data = input.read()
+ if(data) {
+ hash.update(data)
+ } else {
+ resolve(hash.digest('hex'))
+ }
+ })
+ input.on('error', (err) => {
+ reject(err)
+ })
+ })
+
+}
+
+if(require.main == module) {
+ // if run as node process
+ if ( process.argv.length < 4 ) {
+ console.log('ArcGIS (Hosted) Feature Service URL and working directory path required')
+ console.log('example: backuparcgis https://services.arcgis.com/uUvqNMGPm7axC2dD/arcgis/rest/services/state_parks/FeatureServer/0 ./')
+ process.exitCode = 1
+ return
+ }
+
+ const backup = new BackupArcGIS(process.argv[2], process.argv[3])
+ backup.run().then((resp) => {
+ if(!resp.duplicate) {
+ console.log(`${resp.serviceDetails.name} completed: ${resp.filename}`)
+ } else {
+ console.log(`No updates to ${resp.serviceDetails.name}.`)
+ }
+ })
+} else {
+ // if run by require()
+ module.exports = BackupArcGIS
+}
\ No newline at end of file
lib/fs-promise.js 31(+31 -0)
diff --git a/lib/fs-promise.js b/lib/fs-promise.js
new file mode 100644
index 0000000..0c41757
--- /dev/null
+++ b/lib/fs-promise.js
@@ -0,0 +1,31 @@
+const fs = require('fs')
+
+
+function exists(path) {
+ return new Promise(function (resolve, reject) {
+ try {
+ fs.exists(path, (exists) => {
+ resolve(exists)
+ })
+ } catch (err) {
+ reject(err)
+ }
+ })
+}
+
+function readdir(path) {
+ return new Promise(function (resolve, reject) {
+ fs.readdir(path, (err, files) => {
+ if(err) {
+ reject(err)
+ }
+ resolve([...files])
+ })
+ })
+}
+
+// exists, readdir, mkdir, rename, unlink, rename
+
+module.exports = {
+ exists, readdir
+}
lib/https-promise.js 29(+29 -0)
diff --git a/lib/https-promise.js b/lib/https-promise.js
new file mode 100644
index 0000000..6b7281a
--- /dev/null
+++ b/lib/https-promise.js
@@ -0,0 +1,29 @@
+const https = require('https')
+const fs = require('fs')
+const crypto = require('crypto')
+
+function httpsPromise(url, path) {
+ const outfile = fs.createWriteStream(path)
+ return new Promise(function(resolve, reject) {
+ const hash = crypto.createHash('sha256')
+ var req = https.get(url, function (res) {
+ res.pipe(outfile)
+
+ res.on('data', function (d) {
+ hash.update(d)
+ })
+
+ res.on('end', function () {
+ resolve(hash.digest('hex'))
+ })
+ })
+
+ req.on('error', function (err) {
+ reject(err)
+ })
+
+ req.end()
+ })
+}
+
+module.exports = httpsPromise
\ No newline at end of file
package.json 20(+20 -0)
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..353e5db
--- /dev/null
+++ b/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "backuparcgis",
+ "version": "1.0.0",
+ "engines": {
+ "node": ">8.9.0"
+ },
+ "description": "Async module to backup content from ArcGIS Online hosted feature services.",
+ "main": "index.js",
+ "bin": {
+ "backuparcgis": "./index.js"
+ },
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "Joshua Tanner (@Tanner_GEO)",
+ "license": "ISC",
+ "dependencies": {
+ "axios": "^0.18.0"
+ }
+}
package-lock.json 43(+43 -0)
diff --git a/package-lock.json b/package-lock.json
new file mode 100644
index 0000000..7287fc2
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,43 @@
+{
+ "name": "steam_arcgis_2_file.js",
+ "version": "1.0.0",
+ "lockfileVersion": 1,
+ "requires": true,
+ "dependencies": {
+ "axios": {
+ "version": "0.18.0",
+ "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz",
+ "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=",
+ "requires": {
+ "follow-redirects": "1.6.1",
+ "is-buffer": "1.1.6"
+ }
+ },
+ "debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "follow-redirects": {
+ "version": "1.6.1",
+ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.1.tgz",
+ "integrity": "sha512-t2JCjbzxQpWvbhts3l6SH1DKzSrx8a+SsaVf4h6bG4kOXUuPYS/kg2Lr4gQSb7eemaHqJkOThF1BGyjlUkO1GQ==",
+ "requires": {
+ "debug": "3.1.0"
+ }
+ },
+ "is-buffer": {
+ "version": "1.1.6",
+ "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+ "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+ }
+ }
+}
readme.md 73(+73 -0)
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..029debe
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,73 @@
+## BackupArcGIS
+
+Async module to backup content from ArcGIS Online hosted feature services.
+
+## About
+
+Hosted Feature Services in ArcGIS Online are a great way to host and serve geospatial data into your online maps. The infrastructure of ArcGIS Online has proven to be performant and highly available. There is, however, always a chance your mission critical services will run into issues. ArcGIS Online outages, accidental deletion, or service corruption can all happen. It's best to be prepared and at least have a backup of your data on hand.
+
+This library, *BackupArcGIS*, is a performant asynchronous streaming nodejs library that can be imported as a nodejs module or run directly from command line.
+
+## Features
+
+### Asynchronous
+
+BackupArcGIS is asynchronous and returns a promise when complete.
+
+### Versioned
+
+BackupArcGIS only stores a backup of the data if changes has been made. It uses sha256 hashing algorithm to compare new and existing backups. Only services with new data are backed up, saving space.
+
+### Streaming
+
+BackupArcGIS streams feature service content directly to a file and does not store it in memory. This allows the script to run faster and avoid memory issues with larger datasets.
+
+## Use require()
+
+```bash
+$ npm install backuparcgis
+```
+
+```javascript
+const BackupArcGIS = require('backuparcgis')
+
+// new BackupArcGIS(serviceURL, archiveDirectory)
+const Backup = new BackupArcGIS('https://services.arcgis.com/uUvqNMGPm7axC2dD/arcgis/rest/services/state_parks/FeatureServer/0', outDir)
+
+Backup.run().then((resp) => {
+ if(!resp.duplicate) {
+ console.log(`${resp.serviceDetails.name} completed: ${resp.filename}`)
+ } else {
+ console.log(`No updates to ${resp.serviceDetails.name}.`)
+ }
+})
+```
+
+## Run from command line
+
+```bash
+$ npm install backuparcgis
+```
+
+**Format:**
+
+*backuparcgis serviceUrl archiveDirectory*
+
+```bash
+#!/bin/bash
+
+backuparcgis https://services.arcgis.com/uUvqNMGPm7axC2dD/arcgis/rest/services/state_parks/FeatureServer/0 ./terminal
+backuparcgis https://services.arcgis.com/uUvqNMGPm7axC2dD/ArcGIS/rest/services/Brookings_Sites/FeatureServer/0 ./terminal
+```
+
+## Response
+
+The library will respond with a promise with the following object:
+
+```json
+{
+ "duplicate": "boolean",
+ "serviceDetails": "object",
+ "filename": "string"
+}
+```
\ No newline at end of file
test/script.js 41(+41 -0)
diff --git a/test/script.js b/test/script.js
new file mode 100644
index 0000000..41a8cf7
--- /dev/null
+++ b/test/script.js
@@ -0,0 +1,41 @@
+const BackupArcGIS = require('../index.js')
+
+if(require.main == module) {
+ const outDir = './require'
+
+ const Backup1 = new BackupArcGIS('https://services.arcgis.com/uUvqNMGPm7axC2dD/arcgis/rest/services/state_parks/FeatureServer/0', outDir)
+ Backup1.run().then((resp) => {
+ if(!resp.duplicate) {
+ console.log(`${resp.serviceDetails.name} completed: ${resp.filename}`)
+ } else {
+ console.log(`No updates to ${resp.serviceDetails.name}.`)
+ }
+ })
+
+ const Backup2 = new BackupArcGIS('https://services.arcgis.com/uUvqNMGPm7axC2dD/ArcGIS/rest/services/Brookings_Sites/FeatureServer/0', outDir)
+ Backup2.run().then((resp) => {
+ if(!resp.duplicate) {
+ console.log(`${resp.serviceDetails.name} completed: ${resp.filename}`)
+ } else {
+ console.log(`No updates to ${resp.serviceDetails.name}.`)
+ }
+ })
+
+ const Backup3 = new BackupArcGIS('https://services.arcgis.com/uUvqNMGPm7axC2dD/ArcGIS/rest/services/Business_Oregon_Industrial_Sites/FeatureServer/0', outDir)
+ Backup3.run().then((resp) => {
+ if(!resp.duplicate) {
+ console.log(`${resp.serviceDetails.name} completed: ${resp.filename}`)
+ } else {
+ console.log(`No updates to ${resp.serviceDetails.name}.`)
+ }
+ })
+
+ const Backup4 = new BackupArcGIS('https://services.arcgis.com/uUvqNMGPm7axC2dD/ArcGIS/rest/services/OREGON_EDUCATIONAL_BOUNDARIES/FeatureServer/0', outDir)
+ Backup4.run().then((resp) => {
+ if(!resp.duplicate) {
+ console.log(`${resp.serviceDetails.name} completed: ${resp.filename}`)
+ } else {
+ console.log(`No updates to ${resp.serviceDetails.name}.`)
+ }
+ })
+}
\ No newline at end of file
test/terminal.sh 6(+6 -0)
diff --git a/test/terminal.sh b/test/terminal.sh
new file mode 100755
index 0000000..8507467
--- /dev/null
+++ b/test/terminal.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+node ../index.js https://services.arcgis.com/uUvqNMGPm7axC2dD/arcgis/rest/services/state_parks/FeatureServer/0 ./terminal
+node ../index.js https://services.arcgis.com/uUvqNMGPm7axC2dD/ArcGIS/rest/services/Brookings_Sites/FeatureServer/0 ./terminal
+node ../index.js https://services.arcgis.com/uUvqNMGPm7axC2dD/ArcGIS/rest/services/Business_Oregon_Industrial_Sites/FeatureServer/0 ./terminal
+node ../index.js https://services.arcgis.com/uUvqNMGPm7axC2dD/ArcGIS/rest/services/OREGON_EDUCATIONAL_BOUNDARIES/FeatureServer/0 ./terminal
\ No newline at end of file