Merge pull request #3 from kcwongaz/refactor

Refactor

.gitattributes

@@ -0,0 +1 @@
+*.ipynb linguist-language=Python

.gitignore

@@ -1,6 +1,115 @@
-/.vscode/
-/data/
-/figures/
-/sketch/
-/venv/
-/from_others/
+.vscode/
+data/
+raw/
+archive/
+archive_data/
+figures/
+sketch/
+thesis/
+
+
+# --------------------------------------------------------------------------- #
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Environments
+.env
+.venv
+venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 King Chun Wong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,51 @@
+# This Project
+
+Using historical flight trajectory data, can you come up with a quantitive strategy for mitigating flight delay?
+
+This is a project I have worked on with Prof. Michael K. Y. Wong and Prof. Rhea Liem during my MPhil at HKUST. A detailed description of this project can be found in Chapter 4 of my [MPhil thesis](https://drive.google.com/file/d/1wgr3l9psxnW8qiUr-FL-vXN2wRbjcxAC/view?usp=sharing).
+
+This repo contains the code I developed for analyzing historical flight trajectories around the Hong Kong International Airport (HKIA). The flight data used was originally obtained by Prof. Lishuai Li from the City University of Hong Kong and shared with us. Special thanks to Prof. Li for allowing me to release part of the flight data in this repo.
+
+Some example analyses can be found in the notebooks.
+
+<br>
+
+# Getting Started
+
+### 1 - Setting up
+I suggest installing the code locally, e.g.
+
+```bash
+git clone https://github.com/kcwongaz/ust-air-traffic
+cd ust-air-traffic
+pip -e install .  # -e flag for editable mode
+```
+
+This project requires the standard scientific packages, `numpy`, `scipy`, `matplotlib` and `pandas`. In addition, `Cartopy` is needed for drawing maps, and `geopy` is used to compute geodesic distances.
+
+<br>
+
+### 2 - Data
+An example dataset can be downloaded here. 
+
+The example dataset contains the flight data in Jan 2017. Decompressing the data to `data/` at the project root should get the jupyter notebooks running.
+
+If you are interested to see the raw data, here is an example dataset. The raw data is quite large in file size, so I can only provide 3 days of data. To process the raw data, decompress the raw data to `raw/` at the project root, then run
+
+```bash
+. ./pipeline/start.sh 
+```
+
+The scripts in `pipeline/` perform successive processing to prepare the data, e.g. by computing various useful statistics, for further analysis.
+
+<br>
+
+### 3 - The package
+
+Inside `air_traffic/`:
+ - `FR24Writer.py`, `filters.py`:  for processing raw data
+ - `io.py`:  I/O handlers
+ - `loop.py`:  module for analyzing holding patterns and rescheduling
+ - `temporal.py`:  module for analyzing from a time-series perspective
+ - `trajectory.py`:  utility functions for working with flight trajectories
+ - `visual.py`:  utility functions for drawing

preprocessing/FR24Writer.py → air_traffic/FR24Writer.py

@@ -32,7 +32,7 @@ def __init__(self, saveroot):
 
     def set_saveroot(self, saveroot):
         """
-        Reset the saveroot; 
+        Reset the saveroot;
         useful for restoring a old FR24Writer and write data to somewhere else.
         """
         self.saveroot = saveroot

preprocessing/fr24_checkers.py → air_traffic/filters.py

@@ -4,7 +4,7 @@
 
 def check_arrivial(df, last_n=1, tol_lat=0.25, tol_lon=0.25):
     """
-    Check if the trajectory lands around HK, return True if it does. 
+    Check if the trajectory lands around HK, return True if it does.
     """
 
     # Coordinate of HKIA

air_traffic/io.py

@@ -0,0 +1,119 @@
+import pandas as pd
+import numpy as np
+import csv
+import os
+
+import air_traffic.loop as lp
+
+
+def read_trajectories_range(datadir, start, end, verbose=False,
+                            fname_only=False):
+
+    start_date = pd.to_datetime(start)
+    end_date = pd.to_datetime(end)
+    dt = pd.Timedelta(1, "D")
+
+    date = start_date
+    while date <= end_date:
+
+        if verbose:
+            print(f"Now fetching {date}...")
+
+        dstr = date.strftime("%Y-%m-%d")
+        mstr = date.strftime(r"%Y-%m")
+
+        date += dt
+
+        # Skip over months with no data
+        if not os.path.exists(f"{datadir}/{mstr}"):
+            continue
+
+        if fname_only:
+            yield fetch_fnames(datadir, dstr)
+        else:
+            yield read_trajectories(datadir, dstr)
+
+
+def read_trajectories(datadir, date):
+
+    path = os.path.join(datadir, date[:7], date)
+
+    for subdir, _, files in os.walk(path):
+        for file in files:
+            fname = os.path.join(subdir, file)
+            yield pd.read_csv(fname, header=0)
+
+
+def fetch_fnames(datadir, date):
+
+    path = os.path.join(datadir, date[:7], date)
+
+    for subdir, _, files in os.walk(path):
+        for file in files:
+            fname = os.path.join(subdir, file)
+            yield fname
+
+
+def loop_write(dataset, savename):
+
+    with open(savename, mode="w") as f:
+        writer = csv.writer(f, delimiter=',', quotechar='"',
+                            quoting=csv.QUOTE_MINIMAL)
+
+        for dfs in dataset:
+            for df in dfs:
+
+                df = df.loc[df["distance"] <= 200]
+                dist = df["distance"].to_numpy()
+                time = df["time"].to_numpy()
+
+                min_loc = lp.locate_loops(dist)
+
+                _, min_time = lp.find_minima_spacetime(dist, time, min_loc)
+
+                # Skip if no loop is found
+                if len(min_time) == 0:
+                    continue
+
+                _, exit_time = lp.find_exitpoint(dist, time, min_loc)
+
+                # Determine entry direction thus the holding area
+                if df["longitude"].iloc[0] < 114:
+                    area = "CANTO"  # loops in West
+                elif df["latitude"].iloc[0] > 21:
+                    area = "ABBEY"  # loops in Northeast
+                else:
+                    area = "BETTY"  # loops in Southeast
+
+                row = [area]
+                row.extend(min_time)
+                row.append(exit_time)
+                writer.writerow(row)
+
+
+def loop_read(fname, keep_area=True):
+
+    data = []
+    with open(fname) as f:
+        reader = csv.reader(f, delimiter=",")
+        for row in reader:
+
+            # Throw away the loop area label
+            if not keep_area:
+                row = row[1:]
+            data.append(row)
+
+    return data
+
+
+def loop_read_area(fname):
+
+    data_all = loop_read(fname)
+    data = {"ABBEY": [], "BETTY": [], "CANTO": []}
+
+    for entry in data_all:
+        area = entry[0]
+
+        data[area].append(entry[1:])
+
+    return data