###########################################################################
###########################################################################
##
##  Apptainer Build File: Skyply
##  Function: Fully-integrated Simulation stack for WRF, CMAQ, & SCICHEM
##
##  Documentation: 
##
###########################################################################
###########################################################################    

Bootstrap: docker
From: rockylinux:9

%labels
    Author HPCStack
    Version 1.0
    Description "Rocky 9 base with development tools for WRF/CMAQ"

%post
#      ----------------------   Define Relevant Paths and Settings   ------------------
    export HPC_PATH=/opt/hpc
    export MODEL_PATH=/opt/models
    export SOURCE_PATH=/opt/src

    export MPI_HOME=$HPC_PATH/mpi
    export HDF5=$HPC_PATH/hdf5
    export NETCDF=$HPC_PATH/netcdf
    export IOAPI=$HPC_PATH/ioapi

    export PATH=$MPI_HOME/bin:$NETCDF/bin:$PATH
    export LD_LIBRARY_PATH=$MPI_HOME/lib:$HDF5/lib:$NETCDF/lib:$LD_LIBRARY_PATH

    # Compiler wrappers for MPI-enabled builds (used by all packages below)
    # Set these once so each package build uses the same compilers
    export CC=$MPI_HOME/bin/mpicc
    export CXX=$MPI_HOME/bin/mpicxx
    export FC=$MPI_HOME/bin/mpif90

    export CPPFLAGS="-I${HDF5}/include"
    export LDFLAGS="-L${HDF5}/lib"
    export LD_LIBRARY_PATH=${HDF5}/lib:$LD_LIBRARY_PATH

# Update system and enable CRB repository
dnf -y install epel-release
dnf config-manager --set-enabled crb
dnf -y update

# Install development packages
dnf -y install \
    intel-oneapi-compiler-dpcpp-cpp-and-c \
    intel-oneapi-compiler-fortran
#    gcc gcc-gfortran gcc-c++ \
    python3 python3-pip \
    R R-devel \
    git wget which file time \
    make cmake automake autoconf libtool \
    libcurl-devel \
    tar bzip2 bzip2-devel \
    perl tcsh csh m4 \
    jasper jasper-devel \
    libpng libpng-devel \
    zlib zlib-devel \
    libxml2 libxml2-devel \
    xz xz-devel \
    zstd \
    sudo \
    unzip \
    diffutils \
    patch \
    --allowerasing \
    && dnf clean all

mkdir -p ${HPC_PATH} ${MODEL_PATH} ${SOURCE_PATH}

#      ----------------------   Define Package URLs   ------------------
OPENMPI_URL="https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.gz"
HDF5_URL="https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_6/downloads/hdf5-1.14.6.tar.gz"
NETCDF_C_URL="https://downloads.unidata.ucar.edu/netcdf-c/4.10.0/netcdf-c-4.10.0.tar.gz"
NETCDF_FORTRAN_URL="https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.2.tar.gz"
IOAPI_URL="https://www.cmascenter.org/ioapi/download/ioapi-3.2.tar.gz"
WRF_URL="https://github.com/wrf-model/WRF/releases/download/v4.6.0/v4.6.0.tar.gz"
WPS_URL="https://github.com/wrf-model/WPS/archive/refs/tags/v4.6.0.tar.gz"

packages=(
  "openmpi:${OPENMPI_URL}"
  "hdf5:${HDF5_URL}"
  "netcdf-c:${NETCDF_C_URL}"
  "netcdf-fortran:${NETCDF_FORTRAN_URL}"
  "ioapi:${IOAPI_URL}"
  "wrf:${WRF_URL}"
  "wps:${WPS_URL}"
)

#      ----------------------   Download Packages   ------------------
for pkg in "${packages[@]}"; do
    NAME="${pkg%%:*}"
    URL="${pkg#*:}"

    echo "Downloading ${NAME}..."
    
    mkdir -p ${SOURCE_PATH}/${NAME}
    cd ${SOURCE_PATH}/${NAME}
    
    wget ${URL}
done

#      ----------------------   Build OpenMPI   ------------------
    cd ${SOURCE_PATH}/openmpi
    TAR_FILE=$(ls -1t *.tar.gz *.tgz 2>/dev/null | head -n1 || true)
    if [ -n "$TAR_FILE" ]; then
        tar -xzf "$TAR_FILE"
        TOP_DIR=$(tar -tf "$TAR_FILE" | sed -n '1p' | cut -f1 -d"/")
        cd "$TOP_DIR"
    else
        echo "No tarball found in ${SOURCE_PATH}/openmpi"
        exit 1
    fi

    ./configure --prefix=$MPI_HOME
    make -j$(nproc)
    make install

#      ----------------------   Build Parallel HDF5   ------------------

    cd ${SOURCE_PATH}/hdf5

    # get the latest tarball
    TAR_FILE=$(ls -1t *.tar.gz *.tgz 2>/dev/null | head -n1 || true)
    [ -n "$TAR_FILE" ] || { echo "No tarball found"; exit 1; }

    # extract tarball
    tar -xzf "$TAR_FILE"

    # cd into the only directory created (ignore files)
    cd */ || { echo "No directory found after extraction"; exit 1; }

    # configure and install
    ./configure --prefix=$HDF5 --enable-parallel --enable-fortran
    make -j$(nproc)
    make install

#      ----------------------   Build NetCDF-C   ------------------

    cd ${SOURCE_PATH}/netcdf-c
    TAR_FILE=$(ls -1t *.tar.gz *.tgz 2>/dev/null | head -n1 || true)
    if [ -n "$TAR_FILE" ]; then
        tar -xzf "$TAR_FILE"
        TOP_DIR=$(tar -tf "$TAR_FILE" | sed -n '1p' | cut -f1 -d"/")
        cd "$TOP_DIR"
    else
        echo "No tarball found in ${SOURCE_PATH}/netcdf-c"
        exit 1
    fi

    ./configure --prefix=$NETCDF --enable-parallel-tests
    make -j$(nproc)
    make install

#      ----------------------   Build NetCDF-Fortran   ------------------
    cd ${SOURCE_PATH}/netcdf-fortran
    TAR_FILE=$(ls -1t *.tar.gz *.tgz 2>/dev/null | head -n1 || true)
    if [ -n "$TAR_FILE" ]; then
        tar -xzf "$TAR_FILE"
        TOP_DIR=$(tar -tf "$TAR_FILE" | sed -n '1p' | cut -f1 -d"/")
        cd "$TOP_DIR"
    else
        echo "No tarball found in ${SOURCE_PATH}/netcdf-fortran"
        exit 1
    fi

    CPPFLAGS="-I$NETCDF/include" \
    LDFLAGS="-L$NETCDF/lib" \
    ./configure --prefix=$NETCDF --enable-large-files

    make -j$(nproc)
    make install

#      ----------------------   Install WRF   ------------------
    cd ${SOURCE_PATH}/wrf
    TAR_FILE=$(ls -1t *.tar.gz *.tgz 2>/dev/null | head -n1 || true)
    if [ -n "$TAR_FILE" ]; then
        tar -xzf "$TAR_FILE"
        TOP_DIR=$(tar -tf "$TAR_FILE" | sed -n '1p' | cut -f1 -d"/")
        # move whatever was extracted to the models path (wrf)
        mv "$TOP_DIR" ${MODEL_PATH}/wrf
    else
        echo "No tarball found in ${SOURCE_PATH}/wrf"
        exit 1
    fi

    cd ${MODEL_PATH}/wrf

    printf "34\n1\n" | ./configure

    ulimit -s unlimited

    ./compile em_real -j $(nproc) 2>&1 | tee compile.log || true

    ls -lh main || true
    ls -lh run || true

#      ----------------------   Install WPS   ------------------
    cd ${SOURCE_PATH}/wps

    # Extract WPS source and move to models path
    TAR_FILE=$(ls -1t *.tar.gz *.tgz 2>/dev/null | head -n1 || true)
    if [ -n "$TAR_FILE" ]; then
        tar -xzf "$TAR_FILE"
        TOP_DIR=$(tar -tf "$TAR_FILE" | sed -n '1p' | cut -f1 -d"/")
        mv "$TOP_DIR" ${MODEL_PATH}/wps
    else
        echo "No tarball found in ${SOURCE_PATH}/wps"
        exit 1
    fi

    cd ${MODEL_PATH}/wps

    # Tell WPS where WRF is installed
    export WRF_DIR=${MODEL_PATH}/wrf

    # Configure automatically (choose Linux + gfortran)
    printf "1\n" | ./configure

    # Compile WPS
    ./compile 2>&1 | tee compile.log || true

    # Verify binaries
    ls -lh geogrid/src/geogrid.exe || true
    ls -lh ungrib/src/ungrib.exe || true
    ls -lh metgrid/src/metgrid.exe || true

%environment
############################################################
## Compiler Settings
############################################################

# Intel Compiler
export CC=icc
export CXX=icpc
export FC=ifort
export F77=ifort

# GNU Compiler
#export CC=gcc
#export CXX=g++
#export FC=gfortran
#export F77=gfortran

############################################################
## General
############################################################
export HPC_PATH=/opt/hpc
export MODEL_PATH=/opt/models
export SOURCE_PATH=/opt/src


############################################################
## OpenMPI Settings
############################################################
export MPI=/opt/hpc/mpi


############################################################
## OpenMP Settings
############################################################
export OMP_NUM_THREADS=1
export OMP_PROC_BIND=spread
export OMP_PLACES=cores 


############################################################
## HDF5 Settings
############################################################
export HDF5=$HPC_PATH/hdf5
export PHDF5=$HDF5


############################################################
## NetCDF Settings
############################################################
export NETCDF_INCDIR
export NETCDF_LIBDIR
export NETCDF=$HPC_PATH/netcdf
export NETCDF=$NETCDF


############################################################
## IO/API Settings
############################################################
export MPI_HOME=$HPC_PATH/mpi
export IOAPI=$HPC_PATH/ioapi


############################################################
## WRF Settings
############################################################
export WRF_DIR=$MODEL_PATH/wrf
export PATH=$MPI_HOME/bin:$NETCDF/bin:$PATH
export LD_LIBRARY_PATH=$MPI_HOME/lib:$HDF5/lib:$NETCDF/lib:$IOAPI/lib:$LD_LIBRARY_PATH


############################################################
## WPS Settings
############################################################
export WPS_DIR=$MODEL_PATH/wps


############################################################
## CMAQ Settings
############################################################
export CMAQ_DIR=$MODEL_PATH/cmaq
export JASPERLIB=/usr/lib64
export JASPERINC=/usr/include