Migrate from LFS (#75)

Currently the data tables in directory iamdata which are used by some
models are located on LFS. This doesn't allow to test all models on CI.
That should be changed by attaching binary files to release on github.
The tables should be downloaded automatically by python when they are
required.

This PR is opened to solve this problem.

Co-authored-by: Hans Dembinski <hans.dembinski@gmail.com>
Co-authored-by: Hans Dembinski <HDembinski@users.noreply.github.com>
Co-authored-by: afedynitch <afedynitch@gmail.com>
Co-authored-by: Anatoli Fedynitch <afedynitch@users.noreply.github.com>

.github/workflows/release.yml

@@ -86,3 +86,4 @@ jobs:
         with:
           user: __token__
           password: ${{secrets.pypi_password}}
+

.github/workflows/test.yml

@@ -18,13 +18,13 @@ jobs:
       matrix:
         include:
           ## TODO cannot test on windows for now, fortran compiler is not found
-          # - os: windows-latest
-          #   python-version: "3.10"
+          - os: windows-latest
+            python-version: "3.10"
 
           - os: macos-latest
             python-version: "3.7"
 
-          # version number must be string, otherwise 3.10 becomes 3.1
+          # # version number must be string, otherwise 3.10 becomes 3.1
           - os: ubuntu-latest
             python-version: "3.9"
       fail-fast: false
@@ -61,4 +61,24 @@ jobs:
         version: 11
     - run: python -m pip install --upgrade pip
     - run: python -m pip install --prefer-binary -v .[test]
+      if: ${{ matrix.os != 'windows-latest' }}
+
+
+    - if: ${{ matrix.os == 'windows-latest' }}
+      run: |
+        curl -LJO https://github.com/dlfcn-win32/dlfcn-win32/archive/refs/heads/master.zip
+        unzip dlfcn-win32-master.zip
+        python -m pip install cmake
+        cmake -S ./dlfcn-win32-master -B ./build_dlfcn -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release 
+        cmake --build ./build_dlfcn
+        cmake --install ./build_dlfcn 
+
+    - run: python -m pip install --prefer-binary -v .[test]
+      if: ${{ matrix.os == 'windows-latest' }}
+      env:
+        CMAKE_GENERATOR: "MinGW Makefiles"
+        FC: ${{ steps.setup-fortran.outputs.fc }}
+
     - run: python -m pytest -vv
+      env:
+        no_proxy: '*'

.gitignore

@@ -38,4 +38,5 @@ examples/*.root
 tests/*.dat
 extra.cfg
 pythia*.tgz
-impy-?.?.?
+impy-?.?.?
+src/impy/iamdata

CMakeLists.txt

@@ -81,6 +81,7 @@ add_compile_definitions(NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION)
 # generator automatically runs the preprocessor on all Fortran files
 set(CMAKE_Fortran_FLAGS_RELEASE -O3)
 
+
 # CMake sets -fPIC etc. automatically, add only unusual options
 if (UNIX)
   add_compile_options(
@@ -95,7 +96,15 @@ if (UNIX)
     add_compile_options(
       $<$<COMPILE_LANGUAGE:Fortran>:-Wno-argument-mismatch>)
   endif()
-else() # intel
+else() # Windows MinGW!
+	add_compile_options(
+    -Wno-uninitialized
+    -w
+    $<$<COMPILE_LANGUAGE:Fortran>:-std=legacy>
+    $<$<COMPILE_LANGUAGE:Fortran>:-fno-second-underscore>
+	  $<$<COMPILE_LANGUAGE:Fortran>:-fallow-argument-mismatch>
+    $<$<COMPILE_LANGUAGE:CXX>:-fpermissive>
+  )
   # TODO
   # set(FLAGS -fast -fpe0)
   # set(FLAGSF90 ${FLAGS} -ffree-form -Wobsolescent -fno-second-underscore)
@@ -253,6 +262,7 @@ f2py_add_module(_qgsII04
   ${impy_definitions}
 )
 
+if (NOT WIN32)
 ### urqmd34
 set(urqmd34_sources
   1fluid bessel delpart getmass hepcmp iso numrec pythia6409
@@ -284,6 +294,7 @@ f2py_add_module(_urqmd34
   COMPILE_DEFS
   ${impy_definitions}
 )
+endif()
 
 ### pythia6
 f2py_add_module(_pythia6
@@ -324,7 +335,7 @@ f2py_add_module(_pythia6
 # To strengthen this position, we compile the original Pythia-8 code into
 # a shared library to which we only link. This makes clear where the boundary
 # between impy and Pythia-8 is.
-
+if (NOT WIN32)
 file(GLOB pythia8_sources ${cpp_dir}/pythia8307/src/*.cc)
 
 add_library(libpythia8 SHARED ${pythia8_sources})
@@ -335,6 +346,18 @@ target_compile_definitions(libpythia8 PRIVATE XMLDIR="")
 
 pybind11_add_module(_pythia8 ${cpp_dir}/_pythia8.cpp)
 target_link_libraries(_pythia8 PRIVATE libpythia8)
+endif()
+
+
+# This part is needed when we figure out the problem with
+# building Pythia8 on Windows 
+# if (WIN32)
+# find_package(dlfcn-win32 REQUIRED)
+# set(CMAKE_DL_LIBS dlfcn-win32::dl)
+# target_link_libraries(libpythia8 PRIVATE ${CMAKE_DL_LIBS})
+# target_link_libraries(_pythia8 PRIVATE ${CMAKE_DL_LIBS})
+# target_link_libraries(_pythia8 PRIVATE "-static")
+# endif()
 
 ### sophia
 f2py_add_module(_sophia
@@ -412,13 +435,39 @@ list(FILTER dpmjetIII191_sources EXCLUDE REGEX PYR\.f)
 # cmake does not like filenames with parantheses, but some Fortran files
 # include those. As a workaround, we generate modified source files and
 # renamed copies of the headers. Not elegant, but works. 🤷‍♂️
+
+
+### dpmjetIII191
+file(GLOB dpmjetIII191_sources
+  ${fortran_dir}/dpmjetIII-19.1/src/phojet/*.f
+  ${fortran_dir}/dpmjetIII-19.1/src/pythia/*.f
+  ${fortran_dir}/dpmjetIII-19.1/src/dpmjet/*.f
+)
+
+list(FILTER dpmjetIII191_sources EXCLUDE REGEX DT_RNDM\.f)
+list(FILTER dpmjetIII191_sources EXCLUDE REGEX DT_RNDMST\.f)
+list(FILTER dpmjetIII191_sources EXCLUDE REGEX DT_RNDMTE\.f)
+list(FILTER dpmjetIII191_sources EXCLUDE REGEX PYR\.f)
+# cmake does not like filenames with parantheses, but some Fortran files
+# include those. As a workaround, we generate modified source files and
+# renamed copies of the headers. Not elegant, but works. 🤷‍♂️
+
+
+# Workaround to workaround :) Windows don't like long lists
+# User temporary file to keep long lists
+set(temp_dpmjet_file temp.dpmjetIII191)
+string(REPLACE ";"  " " temp_dpmjet_str "${dpmjetIII191_sources}")
+file(WRITE ${temp_dpmjet_file} "${temp_dpmjet_str}")
+
 execute_process(
   COMMAND
-  ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/dpmjetIII191_workaround.py
-  ${CMAKE_CURRENT_BINARY_DIR} ${fortran_dir}/dpmjetIII-19.1/include
-  ${dpmjetIII191_sources}
-  OUTPUT_VARIABLE dpmjetIII191_modded_sources
+  ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/dpmjetIII191_workaround.py 
+  ${CMAKE_CURRENT_BINARY_DIR}
+  ${fortran_dir}/dpmjetIII-19.1/include
+  ${CMAKE_SOURCE_DIR}/${temp_dpmjet_file}
 )
+file(READ ${temp_dpmjet_file} dpmjetIII191_modded_sources)
+file(REMOVE ${temp_dpmjet_file})
 
 
 f2py_add_module(_dpmjetIII191
@@ -439,7 +488,6 @@ f2py_add_module(_dpmjetIII191
   ${fortran_dir}/dpmjetIII-19.1/include/phojet
   ${fortran_dir}/dpmjetIII-19.1/include/dpmjet
   ${fortran_dir}/dpmjetIII-19.1/include/pythia
-  # ${fortran_dir}/dpmjetIII-19.1/include/flinclude
   ${CMAKE_CURRENT_BINARY_DIR}/include
   COMPILE_DEFS
   ${impy_definitions}
@@ -464,7 +512,6 @@ f2py_add_module(_phojet191
   ${fortran_dir}/dpmjetIII-19.1/include/phojet
   ${fortran_dir}/dpmjetIII-19.1/include/dpmjet
   ${fortran_dir}/dpmjetIII-19.1/include/pythia
-  # ${fortran_dir}/dpmjetIII-19.1/include/flinclude
   ${CMAKE_CURRENT_BINARY_DIR}/include
   COMPILE_DEFS
   ${impy_definitions}

F2Py.cmake

@@ -171,6 +171,13 @@ function (f2py_add_module target_name)
     PRIVATE ${F2PY_ADD_MODULE_COMPILE_DEFS})
   endif()  
   target_compile_options(${target_name} PRIVATE -cpp)
+  # Link dll statically in Windows
+  # It can be potentially a PROBLEM!!!
+  # But this is the only way found to build a working library on Windows.
+  # However, on Linux and MacOS, it throws an error at the linking stage.
+  if (WIN32)
+    target_link_libraries(${target_name} PUBLIC "-static")
+  endif()
   set_property(TARGET ${target_name} PROPERTY SUFFIX ${PYTHON_MODULE_EXTENSION})
   # must be a string, so that empty string works correcty
   set_property(TARGET ${target_name} PROPERTY PREFIX "${PYTHON_MODULE_PREFIX}")

cmake_ext.py

@@ -26,13 +26,14 @@
 
 cwd = Path(__file__).parent
 
+
 # Normal print does not work while CMakeBuild is running
 def force_print(msg):
     subp.run(["echo", msg])
 
 
 def cache_value(key, s):
-    m = re.search(key + r":[A-Z]+=([^\s]*)", s)
+    m = re.search(key + r":[A-Z]+=(.*)" + "$", s, flags=re.MULTILINE)
     assert m, f"{key} is not a cached cmake variable"
     return m.group(1)
 
@@ -87,6 +88,7 @@ def build_extension(self, ext):
 
         build_args = ["--config", cfg]  # needed by some generators, e.g. on Windows
 
+        # if cmake_generator in ("Unix Makefiles", "MinGW Makefiles", "MSYS Makefiles"):
         if self.compiler.compiler_type == "msvc":
             # CMake allows an arch-in-generator style for backward compatibility
             contains_arch = any(x in cmake_generator for x in ("ARM", "Win64"))
@@ -102,9 +104,9 @@ def build_extension(self, ext):
                     "win-arm32": "ARM",
                     "win-arm64": "ARM64",
                 }[self.plat_name]
-                cmake_args += ["-A", arch]
+                # cmake_args += ["-A", arch]
 
-            cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"]
+            # cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"]
 
         elif sys.platform.startswith("darwin"):
             # Cross-compile support for macOS - respect ARCHFLAGS if set
@@ -128,22 +130,22 @@ def build_extension(self, ext):
         if cmake_cache.exists():
             with cmake_cache.open() as f:
                 s = f.read()
-                cached_generator = cache_value("CMAKE_GENERATOR", s)
-                cached_cfg = cache_value("CMAKE_BUILD_TYPE", s)
-                disagreement = []
-                for arg in cmake_args:
-                    if arg.startswith("-D"):
-                        key, value = arg[2:].split("=")
-                        cached_value = cache_value(key, s)
-                        disagreement.append(value != cached_value)
-                if any(
-                    disagreement
-                    + [
-                        (cmake_generator and cached_generator != cmake_generator),
-                        cached_cfg != cfg,
-                    ]
-                ):
-                    cmake_cache.unlink()
+            cached_generator = cache_value("CMAKE_GENERATOR", s)
+            cached_cfg = cache_value("CMAKE_BUILD_TYPE", s)
+            disagreement = [
+                (cmake_generator and cached_generator != cmake_generator),
+                cached_cfg != cfg,
+            ]
+            for arg in cmake_args:
+                if arg.startswith("-D"):
+                    key, value = arg[2:].split("=")
+                    cached_value = cache_value(key, s)
+                    # Change \ to / in case of Windows
+                    disagreement.append(
+                        Path(value).absolute() != Path(cached_value).absolute()
+                    )
+            if any(disagreement):
+                cmake_cache.unlink()
 
         # run cmake setup only once
         if not cmake_cache.exists():

dpmjetIII191_workaround.py

@@ -2,7 +2,6 @@
 import re
 import sys
 import shutil
-import os
 
 this_program = Path(sys.argv[0]).name
 
@@ -22,14 +21,16 @@
         for fn in Path(d).rglob("*.f"):
             src.append(fn)
 else:
-    cmake_binary_dir, inc_dir, *src = sys.argv[1:]
+    with Path(sys.argv[3]).open() as f:
+        src = f.read().split()
+    cmake_binary_dir, inc_dir = sys.argv[1:3]
 
 src = [Path(x) for x in src]
 inc_dir = Path(inc_dir)
 cmake_binary_dir = Path(cmake_binary_dir)
 
 dst_include = cmake_binary_dir / "include"
-dst_include.mkdir(exist_ok=True)
+dst_include.mkdir(parents=True, exist_ok=True)
 for fn in inc_dir.rglob("(*)"):
     shutil.copy(fn, dst_include / fn.name[1:-1])
 
@@ -64,4 +65,6 @@
 out = list(map(str, unchanged))
 out += list(map(str, modded))
 
-sys.stdout.write(";".join(out))
+
+with open(sys.argv[3], "w") as f:
+    f.write(";".join(out).replace("\\", "/"))

setup.py

@@ -2,17 +2,20 @@
 from setuptools import setup
 import sys
 import subprocess as subp
+import platform
+import os
 
 cwd = Path(__file__).parent
 
 sys.path.append(str(cwd))
 from cmake_ext import CMakeExtension, CMakeBuild  # noqa: E402
 
-if (cwd / ".git").exists():
-    # make sure that submodules are up-to-date,
-    # it is a common error to forget this when
-    # switching between development branches
-    subp.check_call(["git", "submodule", "update"])
+if not os.environ.get("CI", False):
+    if (cwd / ".git").exists():
+        # make sure that submodules are up-to-date,
+        # it is a common error to forget this when
+        # switching between development branches
+        subp.check_call(["git", "submodule", "update"])
 
 models = [
     "eposlhc",
@@ -23,7 +26,6 @@
     "qgs01",
     "qgsII03",
     "qgsII04",
-    "urqmd34",
     "pythia6",
     "sophia",
     "dpmjet306",
@@ -32,9 +34,13 @@
     "phojet193",
     "dpmjetIII191",
     "dpmjetIII193",
-    "pythia8",
 ]
 
+# urqmd34 doesn't build correctly on Windows
+if platform.system() != "Windows":
+    models.append("urqmd34")
+    models.append("pythia8")
+
 # for convenience, support building extra models via extra.cfg
 # extra.cfg is not tracked by git, so can be freely modified
 # extra.cfg example:

src/f2py/_eposlhc-f2pywrappers.f

@@ -1,5 +1,5 @@
 C     -*- fortran -*-
-C     This file is autogenerated with f2py (version:1.23.1)
+C     This file is autogenerated with f2py (version:2)
 C     It contains Fortran 77 wrappers to fortran functions.
 
       subroutine f2pywrapidtrafo (idtrafof2pywrap, code1, code2, i

src/f2py/_eposlhc.pyf

@@ -8956,6 +8956,5 @@ python module _eposlhc ! in
     end interface 
 end python module _eposlhc
 
-! This file was auto-generated with f2py (version:1.23.1).
-! See:
-! https://web.archive.org/web/20140822061353/http://cens.ioc.ee/projects/f2py2e
+! This file was auto-generated with f2py (version:2).
+! See http://cens.ioc.ee/projects/f2py2e/