automating database population

database_inflater.py

@@ -15,14 +15,14 @@ def push_into_table(db_name, username, password, table_name, data):
         cursor = conn.cursor()  # cursors give us API for executing SQL queries
         count = 0
         for feature_id, feature_name, outline in data:
-            cursor.execute('insert into %s values ( %s, %s, %s)',
+            cursor.execute('insert into %s values ( %s, %s, st_geogfromtext(%s))',
                            (table_name, feature_id, feature_name, outline))
             count += 1
             if(not count % 100):
                 # after every 100 push, we just commit the changes made
                 conn.commit()
-        cursor.close()
         conn.commit()  # final commit
+        cursor.close()
         conn.close()  # connection to DB closed
     except psql.DatabaseError as e:
         print('[!]Error: {}'.format(str(e)))
@@ -38,13 +38,13 @@ def create_table(db_name, username, password, table_name):
     try:
         conn = psql.connect(database=db_name, user=username, password=password)
         cursor = conn.cursor()
-        cursor.execute('drop table %s if exists', (table_name,))
+        cursor.execute('drop table if exists %s', (table_name,))
         cursor.execute(
             'create table %s (feature_id varchar primary key, feature_name varchar not null, outline geography)', (table_name, ))
         cursor.execute('create index %s on %s using gist( outline )',
                        ('{}_index'.format(table_name), table_name))
-        cursor.close()
         conn.commit()  # committing changes made to DB
+        cursor.close()
         conn.close()  # closed connection to DB
     except psql.DatabaseError as e:
         print('[!]Error: {}'.format(str(e)))
@@ -56,14 +56,18 @@ def create_table(db_name, username, password, table_name):
 
 
 def inflate_into_db(db_name, username, password, data_set):
+    # database inflation handler
     try:
         for key, value in data_set.items():
             if(create_table(db_name, username, password, 'world_features_level_{}'.format(key))):
+                print(
+                    '[+]Pushing into table -- `{}`'.format('world_features_level_{}'.format(key)))
                 push_into_table(db_name, username, password,
                                 'world_features_level_{}'.format(key), value)
     except Exception as e:
         print('[!]Error: {}'.format(str(e)))
         return False
+    return True
 
 
 if __name__ == '__main__':

download_gadm_shape_files_with_layers_seperated.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/bash
+
+# script downloads shape files from GADM, make sure you're connected to internet
+wget https://biogeo.ucdavis.edu/data/gadm3.6/gadm36_levels_shp.zip
+# unzips downloaded zip into multiple layered shape files, which will be later on used for inflating features into database
+unzip gadm36_levels_shp.zip

fetch_and_push.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+try:
+    from database_inflater import inflate_into_db
+    from osgeo import ogr as geo
+    from subprocess import run
+except ImportError as e:
+    print('[!]Module Unavailable : {}'.format(str(e)))
+    exit(1)
+
+
+def app(path='/home/anjan/Documents/my_programs/still_working/python_codes/gis_data/gadm_world_data/gadm36_{}.shp', file_id=[0, 1, 2, 3, 4, 5]):
+    # path, path to gadm shapefiles
+    # gadm has 6 layers, shape files hold corresponding layer number too
+    data_set = {}
+    for i in file_id:
+        print('[+]Working on `{}`'.format(path.format(i)))
+        datasource = geo.Open(path.format(i))  # datasource opened
+        # layer fetched, only a single layer present in a shape file
+        layer = datasource.GetLayer(0)
+        tmp = []
+        for j in range(layer.GetFeatureCount()):
+            feature = layer.GetFeature(j)  # gets feature by id
+            tmp.append([feature.items().get('GID_{}'.format(i)),
+                        feature.items().get('NAME_{}'.format(i)),
+                        feature.GetGeometryRef().ExportToWkt()])
+            # holds data in temp variable
+            # data format -- [feature_id, feature_name, outline]
+        data_set.update({i: tmp})  # pushes into dictionary
+    print(data_set)
+    return
+    if(inflate_into_db('world_features', 'postgres', '@njan5m3dB', data_set)):
+        # finally inflate into database
+        print('[+]Done')
+    return
+
+
+if __name__ == '__main__':
+    try:
+        run('clear')
+        app()
+    except KeyboardInterrupt:
+        print('\n[!]Terminated')
+    finally:
+        exit(0)