diff --git a/.gitignore b/.gitignore index bfd2475..328e5ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ __pycache__ venv/ .idea/ -venvrimo3/ rinexmod.egg-info/ +py3rimo4a/ +venvrimo3/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..540b720 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt \ No newline at end of file diff --git a/README.md b/README.md index 8b3ba58..e816a87 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ It is developed in Python 3, and can be run from the command line or directly in The required input metadata can come from a sitelog file, or be manually entered as arguments to the command line or the called function. It is available under the GNU license on the following GitHub repository: https://github.com/IPGP/rinexmod -v2 - 2023-05-15 - Pierre Sakic - sakic@ipgp.fr -v1 - 2022-02-07 - Félix Léger - leger@ipgp.fr +v2+ - 2023-05-15 - Pierre Sakic - sakic@ipgp.fr +v1 - 2022-02-07 - Félix Léger - leger@ipgp.fr Last version: v3.4.0 - 2024-09-30 @@ -17,7 +17,7 @@ Last version: v3.4.0 - 2024-09-30 ### Main tool -* `rinexmod.py` takes a list of RINEX Hatanaka compressed files (.d.Z or .d.gz or .rnx.gz), +* `rinexmod_run` takes a list of RINEX Hatanaka compressed files (.d.Z or .d.gz or .rnx.gz), loops the rinex files list to modify the file's headers. It then writes them back to Hatanaka compressed format in an output folder. It also permits to rename the files, changing the four first characters of the file name with another station code. It can write @@ -25,7 +25,7 @@ those files with the long name naming convention with the --longname option. ### Annex tools -They are stored in `misc_tools` folder. +They are stored in `bin/misc_tools` folder. * `get_m3g_sitelogs.py` will get the last version of site logs from the M3G repository and write them in an observatory-dependent subfolder. @@ -42,14 +42,7 @@ The tool is designed in Python 3, and you must have it installed on your machine You can use `pip` to install the last GitHub-hosted version with the following command: ```pip install git+https://github.com/IPGP/rinexmod``` -To use the front-end functions, you must also add the `rinexmod` folder in your `$PATH` environnement variable, -defined in your `.bashrc`. -If you used PIP for the installation, your can locate the folder where `rinexmod` has been installed with: -``` -pip show rinexmod -``` -at the line `Location:`. Usually, it is `/usr/local/lib/python3.NN/dist-packages/rinexmod` . -Then edit your `~/.bashrc` and add it to the `$PATH` environnement variable +Since the version 3.4.0, the frontend program `rinexmod_run` is available directly when you call it in your console. ### Required external modules @@ -67,11 +60,9 @@ You can install them with: pip install hatanaka pycountry matplotlib colorlog pandas ``` - - ## _rinexmod_ in command lines interface -### rinexmod.py +### rinexmod_run This is the main frontend function. It takes a list of RINEX Hatanaka compressed files (.d.Z or .d.gz or .rnx.gz), loop over the RINEX files list to modify the file's header. It then writes them back to Hatanaka-compressed @@ -146,13 +137,13 @@ _rinexmod_ will add two comment lines, one indicating the source of the modifica ### Synopsis ``` -rinexmod.py [-h] -i RINEXINPUT [RINEXINPUT ...] -o OUTPUTFOLDER - [-s SITELOG] [-k KEY=VALUE [KEY=VALUE ...]] [-m MARKER] - [-co COUNTRY] [-n NINECHARFILE] [-sti STATION_INFO] - [-lfi LFILE_APRIORI] [-r RELATIVE] [-nh] [-c {gz,Z,none}] - [-l] [-fs] [-fc] [-fr] [-ig] [-a] [-ol OUTPUT_LOGS] [-w] - [-v] [-t] [-u] [-fns {basic,flex,exact}] - [-mp MULTI_PROCESS] [-d] [-rm] +rinexmod_run [-h] -i RINEXINPUT [RINEXINPUT ...] -o OUTPUTFOLDER + [-s SITELOG] [-k KEY=VALUE [KEY=VALUE ...]] [-m MARKER] + [-co COUNTRY] [-n NINECHARFILE] [-sti STATION_INFO] + [-lfi LFILE_APRIORI] [-r RELATIVE] [-nh] [-c {gz,Z,none}] + [-l] [-fs] [-fc] [-fr] [-ig] [-a] [-ol OUTPUT_LOGS] [-w] + [-v] [-t] [-u] [-fns {basic,flex,exact}] + [-mp MULTI_PROCESS] [-d] [-rm] RinexMod takes RINEX files (v2 or v3/4, compressed or not), rename them and modifiy their headers, and write them back to a destination directory @@ -195,7 +186,7 @@ optional arguments: -fs, --force_sitelog If a single sitelog is provided, force sitelog-based header values when RINEX's header and sitelog site name do not correspond. If several sitelogs are provided, skip badly-formated sitelogs. -fc, --force_fake_coords - When using GAMIT station.info metadata without apriori coordinates in the L-File, gives fake coordinates at (0°,0°) to the site + When using GAMIT station.info metadata without apriori coordinates in the L-File, gives fake coordinates at (0??,0??) to the site -fr, --force_rnx_load Force the loading of the input RINEX. Useful if its name is not standard -ig, --ignore Ignore firmware changes between instrumentation periods when getting header values info from sitelogs @@ -229,10 +220,10 @@ RinexMod 3.3.0 - GNU Public Licence v3 - P. Sakic et al. - IPGP-OVS - https://gi ### Examples ``` -./rinexmod.py -i RINEXLIST -o OUTPUTFOLDER (-k antenna_type='ANT TYPE' antenna_X_pos=9999 agency=AGN) (-m AGAL) (-r ./ROOTFOLDER/) (-f) (-v) +./rinexmod_run -i RINEXLIST -o OUTPUTFOLDER (-k antenna_type='ANT TYPE' antenna_X_pos=9999 agency=AGN) (-m AGAL) (-r ./ROOTFOLDER/) (-f) (-v) ``` ``` -./rinexmod.py (-a) -i RINEXFILE -o OUTPUTFOLDER (-s ./sitelogsfolder/stationsitelog.log) (-i) (-w) (-o ./LOGFOLDER) (-v) +./rinexmod_run (-a) -i RINEXFILE -o OUTPUTFOLDER (-s ./sitelogsfolder/stationsitelog.log) (-i) (-w) (-o ./LOGFOLDER) (-v) ``` ## _rinexmod_ in API mode @@ -373,7 +364,7 @@ rimo_api.rinexmod(rinexfile, outputfolder, sitelog=None, modif_kw=dict(), marker position and DOMES information (needs also station_info option) force_fake_coords: bool, optional When using GAMIT station.info metadata without apriori coordinates - in the L-File, gives fake coordinates at (0°,0°) to the site + in the L-File, gives fake coordinates at (0??,0??) to the site remove: bool, optional Remove input RINEX file if the output RINEX is correctly written The default is False. @@ -397,17 +388,17 @@ rimo_api.rinexmod(rinexfile, outputfolder, sitelog=None, modif_kw=dict(), marker ``` ## Other command line functions -### crzmeta.py +### crzmeta Extract metadata from crz file. With -p option, will plot the file's samples intervals ``` EXAMPLE: -./crzmeta.py RINEXFILE (-p) +./crzmeta RINEXFILE (-p) ``` -### get_m3g_sitelogs.py +### get_m3g_sitelogs This script will get the last version of sitelogs from M3G repository and write them in an observatory dependent subfolder set in 'observatories'. @@ -425,7 +416,7 @@ OPTION : EXAMPLE: - ./get_m3g_sitelogs.py OUTPUTFOLDER (-d) + ./get_m3g_sitelogs OUTPUTFOLDER (-d) ``` ## _rinexmod_ error messages diff --git a/changelog.md b/changelog.md index ce61953..28e5fb6 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,34 @@ ## Changelog +### v3.4.0 (Dec, 2024) + +- shell programs are now in a `bin` folder and are set as executable in the `setup.py` +- **the frontend command line interface program has been renamed `rinexmod_run.py` to avoid conflict with the module name** +- add `shortname` option to force RINEX file renaming with short name convention, and modify `longname` option to force RINEX file renaming with long name convention + (Behavior of previous `longname` option was ambiguous.) +`shortname` is mutually exclusive with `longname`. + +### v3.3.0 (Sep 4, 2024) + +* `--tolerant_file_period/-tol` is replaced by a more flexible --filename_style/-fns option. + * it takes now one string argument with 3 acceptable values: 'basic' (per default), 'flex', and 'exact'. + see help for more details + * misc. improvements and bug corrections + +### v3.2.0 (Aug 15, 2024) + +Misc improvements: + +* Enhanced log messages +* Some exception handling when reading RINEX +* RINEX version attribute is now also float +* you can now remove your input RINEXs with -rm, but be carful of what you are doing! + + +### v3.1.0 (Jun 5, 2024) + +make rinexmod compliant with IGSMAIL-8458, i.e. nine characters site in sitelogs, and Country or Region field instead of Country. + ### v3.0.0 (Mar 15, 2024) - **important user interface change** diff --git a/requirements.txt b/requirements.txt index 12c46e7..5d5f4c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ colorlog pandas hatanaka -matplotlib +#matplotlib numpy requests setuptools \ No newline at end of file diff --git a/rinexmod/bin/__init__.py b/rinexmod/bin/__init__.py new file mode 100644 index 0000000..d1e12f0 --- /dev/null +++ b/rinexmod/bin/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on 21/09/2024 20:30:36 + +@author: psakic +""" diff --git a/rinexmod/bin/misc_tools/__init__.py b/rinexmod/bin/misc_tools/__init__.py new file mode 100644 index 0000000..d1e12f0 --- /dev/null +++ b/rinexmod/bin/misc_tools/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on 21/09/2024 20:30:36 + +@author: psakic +""" diff --git a/misc_tools/crzmeta.py b/rinexmod/bin/misc_tools/crzmeta.py similarity index 98% rename from misc_tools/crzmeta.py rename to rinexmod/bin/misc_tools/crzmeta.py index 08c0e5b..9ae465e 100755 --- a/misc_tools/crzmeta.py +++ b/rinexmod/bin/misc_tools/crzmeta.py @@ -54,8 +54,7 @@ def crzmeta(rinexfile, plot): return -if __name__ == '__main__': - +def main(): import argparse # Parsing Args @@ -68,3 +67,6 @@ def crzmeta(rinexfile, plot): plot = args.plot crzmeta(rinexfile, plot) + +if __name__ == '__main__': + main() diff --git a/misc_tools/get_m3g_sitelogs.py b/rinexmod/bin/misc_tools/get_m3g_sitelogs.py similarity index 98% rename from misc_tools/get_m3g_sitelogs.py rename to rinexmod/bin/misc_tools/get_m3g_sitelogs.py index 0c3e970..0b2ba83 100755 --- a/misc_tools/get_m3g_sitelogs.py +++ b/rinexmod/bin/misc_tools/get_m3g_sitelogs.py @@ -47,8 +47,7 @@ import rinexmod.get_m3g -if __name__ == "__main__": - +def main(): import argparse # Parsing Args @@ -123,3 +122,6 @@ rinexmod.get_m3g.get_m3g_sitelogs( sitelogsfolder, delete, observatory, root, svn, move_folder, force, exclude ) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/misc_tools/rinexrename.py b/rinexmod/bin/misc_tools/rinexrename.py similarity index 99% rename from misc_tools/rinexrename.py rename to rinexmod/bin/misc_tools/rinexrename.py index dcdf5f6..3d19030 100755 --- a/misc_tools/rinexrename.py +++ b/rinexmod/bin/misc_tools/rinexrename.py @@ -58,8 +58,7 @@ def rinexrename(rinexinput, output=None, delete=False, alone=False, country="00X return Output_path_list -if __name__ == '__main__': - +def main(): import argparse @@ -80,3 +79,6 @@ def rinexrename(rinexinput, output=None, delete=False, alone=False, country="00X country = args.country rinexrename(rinexinput, output, delete, alone, country) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/rinexmod.py b/rinexmod/bin/rinexmod_run.py similarity index 99% rename from rinexmod.py rename to rinexmod/bin/rinexmod_run.py index 67220d2..c18a649 100755 --- a/rinexmod.py +++ b/rinexmod/bin/rinexmod_run.py @@ -16,15 +16,12 @@ import argparse import textwrap -import textwrap - - import rinexmod import rinexmod.rinexmod_api as rimo_api from argparse import RawTextHelpFormatter -if __name__ == "__main__": +def main(): class SmartFormatter(argparse.HelpFormatter): # source: https://stackoverflow.com/a/22157136/3464212 diff --git a/rinexmod/rinexfile.py b/rinexmod/rinexfile.py index 68265a4..60715c0 100644 --- a/rinexmod/rinexfile.py +++ b/rinexmod/rinexfile.py @@ -13,7 +13,7 @@ from pathlib import Path import hatanaka -import matplotlib.pyplot as plt +#import matplotlib.pyplot as plt import numpy as np import rinexmod.logger as rimo_log @@ -675,14 +675,14 @@ def _get_date_patterns(self): # Pattern of an observation line containing a date - RINEX 3 # date_pattern = re.compile('> (\d{4}) (\d{2}) (\d{2}) (\d{2}) (\d{2}) ((?: |\d)\d.\d{4})') date_pattern = re.compile( - "> (\d{4}) (\d{2}| \d) (\d{2}| \d) (\d{2}| \d) (\d{2}| \d) ((?: |\d)\d.\d{4})" + r"> (\d{4}) (\d{2}| \d) (\d{2}| \d) (\d{2}| \d) (\d{2}| \d) ((?: |\d)\d.\d{4})" ) year_prefix = "" # Prefix of year for date formatting elif self.version_float < 3: # Pattern of an observation line containing a date - RINEX 2 date_pattern = re.compile( - " (\d{2}) ((?: |\d)\d{1}) ((?: |\d)\d{1}) ((?: |\d)\d{1}) ((?: |\d)\d{1}) ((?: |\d)\d{1}.\d{4})" + r" (\d{2}) ((?: |\d)\d{1}) ((?: |\d)\d{1}) ((?: |\d)\d{1}) ((?: |\d)\d{1}) ((?: |\d)\d{1}.\d{4})" ) year_prefix = "20" # Prefix of year for date formatting ### !!!!!!!!! before 2000 must be implemented !!!!!! @@ -881,7 +881,9 @@ def _date_conv(sample): non_nominal_interval_percent = num_bad_sp / len(samples_rate_diff) + plot = False if plot: + import matplotlib.pyplot as plt print( "{:29} : {}".format( "Sample intervals not nominals", @@ -2164,13 +2166,13 @@ def regex_pattern_rinex_filename(): pattern_dic = dict() # pattern_dic["shortname"] = "....[0-9]{3}(\d|\D)\.[0-9]{2}(o|d)(|\.(Z|gz))" pattern_dic["shortname"] = ( - "....[0-9]{3}(\d|\D)([0-9]{2}\.|\.)[0-9]{2}(o|d)(|\.(Z|gz))" ### add subhour starting min + r"....[0-9]{3}(\d|\D)([0-9]{2}\.|\.)[0-9]{2}(o|d)(|\.(Z|gz))" ### add subhour starting min ) pattern_dic["longname"] = ( - ".{4}[0-9]{2}.{3}_(R|S|U)_[0-9]{11}_([0-9]{2}\w)_[0-9]{2}\w_\w{2}\.\w{3}(\.gz|)" + r".{4}[0-9]{2}.{3}_(R|S|U)_[0-9]{11}_([0-9]{2}\w)_[0-9]{2}\w_\w{2}\.\w{3}(\.gz|)" ) pattern_dic["longname_gfz"] = ( - ".{4}[0-9]{2}.{3}_[0-9]{8}_.{3}_.{3}_.{2}_[0-9]{8}_[0-9]{6}_[0-9]{2}\w_[0-9]{2}\w_[A-Z]*\.\w{3}(\.gz)?" + r".{4}[0-9]{2}.{3}_[0-9]{8}_.{3}_.{3}_.{2}_[0-9]{8}_[0-9]{6}_[0-9]{2}\w_[0-9]{2}\w_[A-Z]*\.\w{3}(\.gz)?" ) return pattern_dic diff --git a/rinexmod/rinexmod_api.py b/rinexmod/rinexmod_api.py index cb7046f..4f7df36 100755 --- a/rinexmod/rinexmod_api.py +++ b/rinexmod/rinexmod_api.py @@ -18,6 +18,7 @@ import hatanaka import pandas as pd +import rinexmod as rimo import rinexmod.gamit_meta as rimo_gmm import rinexmod.logger as rimo_log import rinexmod.metadata as rimo_mda @@ -67,14 +68,13 @@ def listfiles(directory, extension, recursive=True): # get Git hash (to get a version number-equivalent of the RinexMod used) -def git_get_revision_short_hash(): +def get_git_hash(): """ Gives the Git hash to have a tracking of the used version Returns ------- 7 characters Git hash - """ script_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -86,7 +86,6 @@ def git_get_revision_short_hash(): githash = "xxxxxxx" ####NB: 2msec to run this fuction - return githash @@ -269,7 +268,7 @@ def sitelogs2metadata_objs( sitelog_extension = ".log" all_sitelogs = listfiles(sitelog_filepath, sitelog_extension) - sitelog_pattern = re.compile("\w{4,9}_\d{8}.log") + sitelog_pattern = re.compile(r"\w{4,9}_\d{8}.log") all_sitelogs = [ file for file in all_sitelogs @@ -1195,7 +1194,8 @@ def rinexmod( ########################################################################### ########## Add comment in the header - vers_num = git_get_revision_short_hash() + githash = get_git_hash() + vers_num = rimo.__version__ + " " + githash[-3:] # rnxobj.add_comment(("RinexMod (IPGP)","METADATA UPDATE"),add_pgm_cmt=True) rnxobj.add_prg_run_date_comment("RinexMod " + vers_num, "METADATA UPDATE") rnxobj.add_comment("RinexMod / IPGP-OVS (github.com/IPGP/rinexmod)") @@ -1444,6 +1444,10 @@ def rinexmod_cli( else: rinexinput_use = rinexinput + if not rinexinput_use: + logger.error("The input file is empty: %s", str(rinexinput)) + return RinexModInputArgsError + if rinexinput_use[0].endswith("RINEX VERSION / TYPE"): logger.error( "The input file is not a file list but a RINEX: %s", str(rinexinput[0]) @@ -1574,4 +1578,4 @@ def _print_kw_tips(values_inp): raise e -# ***************************************************************************** +# ***************************************************************************** \ No newline at end of file diff --git a/setup.py b/setup.py index a4f6608..69a9f66 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,13 @@ # Get the long description from the README file long_description = (here / 'README.md').read_text(encoding='utf-8') + +# Load the requirements from the requirements.txt file +# Frederick Brennan answer on stackoverflow +# https://stackoverflow.com/questions/14399534/reference-requirements-txt-for-the-install-requires-kwarg-in-setuptools-setup-py +with open(here / 'requirements.txt') as f: + required = f.read().splitlines() + # Arguments marked as "Required" below must be included for upload to PyPI. # Fields marked as "Optional" may be commented out. @@ -39,7 +46,7 @@ # project code, see # https://packaging.python.org/en/latest/single_source_version.html - version=str('3.4.0'), # Required ## CHANGE IT ALSO IN __init____ and readme !!!! + version=str('3.4.0'), # Required ## CHANGE IT ALSO IN __init__ and readme !!!! # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: @@ -114,7 +121,6 @@ # 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3 :: Only', ], - # This field adds keywords for your project which will appear on the # project page. What does your project relate to? # @@ -150,13 +156,10 @@ # # For an analysis of "install_requires" vs pip's requirements files see: # https://packaging.python.org/en/latest/requirements.html - install_requires=[ 'hatanaka', - 'numpy', - 'pycountry', - 'matplotlib', - 'colorlog', - 'pandas', - ], # Optional + + # requirements.txt is read at the beginning of the setup.py file + # and the required packages are stored in the list 'required' + install_requires=required, # Optional # List additional groups of dependencies here (e.g. development # dependencies). Users will be able to install these using the "extras" @@ -173,7 +176,7 @@ # 'kepler.py', # 'ncompress'], - } + }, # If there are data files included in your packages that need to be # installed, specify them here. @@ -195,11 +198,19 @@ # # For example, the following would provide a command called `sample` which # executes the function `main` from this package when invoked: - #entry_points={ # Optional - # 'console_scripts': [ - # 'sample=sample:main', - # ], - #}, + entry_points={ # Optional + 'console_scripts': [ + 'rinexmod_run=rinexmod.bin.rinexmod_run:main', + 'crzmeta=rinexmod.bin.misc_tools.crzmeta:main', + 'get_m3g_sitelogs=rinexmod.bin.misc_tools.get_m3g_sitelogs:main', + 'rinexrename=rinexmod.bin.misc_tools.rinexrename:main' + ], + }, + + # scripts = ['bin/rinexmod_run.py', + # 'bin/misc_tools/rinexrename.py', + # 'bin/misc_tools/get_m3g_sitelogs.py', + # 'bin/misc_tools/crzmeta.py'], # List additional URLs that are relevant to your project as a dict. #