sync-socrata-airtable/scratch.js at master · DataSF/sync-socrata-airtable · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
// To capture public vs. private, we must query the Discovery API using the public param twice

/*
syncDiscoveryAPI(
  'https://api.us.socrata.com/api/catalog/v1?domains=data.sfgov.org&search_context=data.sfgov.org&provenance=official&limit=100&public=true&', transformDiscoveryPublic,
  () => {
    syncDiscoveryAPI(
      'https://api.us.socrata.com/api/catalog/v1?domains=data.sfgov.org&search_context=data.sfgov.org&provenance=official&limit=100&public=false&', transformDiscoveryPrivate,
      () => {
        syncMetadataAPI(syncDictionaryAPI)
      })
})
*/
/*
syncDatasetProfiles()

export function syncDiscoveryAPI (url, transform, cb) {
  syncDatasets(
    url,
    {
      pageParam: 'offset',
      page: 0,
      add: 100,
      transform: transform
    },
    cb
  )
}

export function syncMetadataAPI (cb) {
  syncDatasets('https://data.sfgov.org/api/views/metadata/v1?limit=100&',
  {
    pageParam: 'page',
    page: 1,
    add: 1,
    transform: transformMetadata
  }, () => {
    cb()
  })
}

export function syncDictionaryAPI () {
  syncDatasets('https://data.sfgov.org/resource/cq5k-ka7d.json?$select=datasetid,SUM(CASE(field_documented=true,1)),SUM(CASE(field_documented=false,0)),count(*)&$group=datasetid&$limit=100&$order=datasetid&',
  {
    pageParam: '$offset',
    page: 0,
    add: 100,
    transform: transformDocumentation
  }, () => {
    console.log('Done loading assets from master data dictionary')
    syncDatasetProfiles()
  })
}

export function syncDatasetProfiles () {
  syncDatasets('https://data.sfgov.org/resource/8ez2-fksg.json?&$limit=100&$order=datasetid&',
{
  pageParam: '$offset',
  page: 0,
  add: 100,
  transform: transformProfiles
}, () => {
  console.log('Done loading assets from asset inventory')
  })
}
*/

/*
syncDatasetsAsync('https://api.us.socrata.com/api/catalog/v1?domains=data.sfgov.org&search_context=data.sfgov.org&provenance=official&limit=3&public=true&',
  {
    pageParam: 'offset',
    page: 0,
    add: 3,
    transform: transformDiscoveryPublic
  }).then(() => {
    syncDatasetsAsync('https://api.us.socrata.com/api/catalog/v1?domains=data.sfgov.org&search_context=data.sfgov.org&provenance=official&limit=3&public=false&',
    {
      pageParam: 'offset',
      page: 0,
      add: 3,
      transform: transformDiscoveryPrivate
    })
  }).then(() => {

  })
  */