Merge pull request #41 from holukas/indev

Add scatter plot

CHANGELOG.md

@@ -2,11 +2,17 @@
 
 ![DIIVE](images/logo_diive1_256px.png)
 
-## v0.66.0 | X Nov 2023
+## v0.66.0 | 2 Nov 2023
 
-### XXX
+### New features
+
+- Added new class `ScatterXY`: a simple scatter plot that supports bins (`core.plotting.scatter.ScatterXY`)
+
+![DIIVE](images/ScatterXY_diive_v0.66.0.png)
+
+### Notebooks
 
-- XXX
+- Added notebook `notebooks/Plotting/ScatterXY.ipynb`
 
 ## v0.64.0 | 31 Oct 2023
 

README.md

@@ -100,6 +100,7 @@ Fill gaps in time series with various methods
 ### Plotting
 
 - Simple (interactive) time series plot ([notebook example](notebooks/Plotting/TimeSeries.ipynb))
+- ScatterXY plot ([notebook example](notebooks/Plotting/ScatterXY.ipynb))
 - Various classes to generate heatmaps, bar plots, time series plots and scatter plots, among others
 
 ### Quality control

diive/core/plotting/plotfuncs.py

@@ -90,7 +90,8 @@ def default_format(ax,
 
     # Labels
     if ax_xlabel_txt:
-        ax.set_xlabel(ax_xlabel_txt, color=ax_labels_fontcolor, fontsize=ax_labels_fontsize, fontweight=ax_labels_fontweight)
+        ax.set_xlabel(ax_xlabel_txt, color=ax_labels_fontcolor, fontsize=ax_labels_fontsize,
+                      fontweight=ax_labels_fontweight)
     if ax_ylabel_txt and txt_ylabel_units:
         ax.set_ylabel(f'{ax_ylabel_txt}  {txt_ylabel_units}', color=ax_labels_fontcolor, fontsize=ax_labels_fontsize,
                       fontweight=ax_labels_fontweight)
@@ -314,11 +315,13 @@ def default_legend(ax,
                            ncol=ncol, facecolor=facecolor, edgecolor=edgecolor,
                            labelspacing=labelspacing, prop={'size': textsize},
                            markerscale=markerscale)
+
     else:
         legend = ax.legend(loc=loc, bbox_to_anchor=bbox_to_anchor, shadow=shadow,
                            ncol=ncol, facecolor=facecolor, edgecolor=edgecolor,
                            labelspacing=labelspacing, prop={'size': textsize},
                            markerscale=markerscale)
+
     for text in legend.get_texts():
         text.set_color(textcolor)
 
@@ -499,10 +502,12 @@ def save_fig(fig,
     print(f"Saved plot {outfilepath}")
 
 
-def create_ax():
+def create_ax(facecolor: str = 'white',
+              figsize: tuple = (8, 4.5),
+              dpi: int = 100):
     """Create figure and axis"""
     # Figure setup
-    fig = plt.figure(facecolor='white', figsize=(16, 9))
+    fig = plt.figure(facecolor=facecolor, figsize=figsize, dpi=dpi)
     gs = gridspec.GridSpec(1, 1)  # rows, cols
     # gs.update(wspace=0.3, hspace=0.3, left=0.03, right=0.97, top=0.97, bottom=0.03)
     ax = fig.add_subplot(gs[0, 0])

diive/core/plotting/scatter.py

@@ -1,76 +1,112 @@
+import matplotlib.pyplot as plt
 import pandas as pd
 from pandas import Series
 
 import diive.core.plotting.plotfuncs as pf
-import diive.core.plotting.styles.LightTheme as theme
-
-
-# TODO
-# TODO
-# TODO
-# TODO
-# TODO
-
-class Scatter:
-    def __init__(self,
-                 x: Series,
-                 y: Series,
-                 xunits: str = None,
-                 yunits: str = None,
-                 title: str = None
-                 ):
+
+
+class ScatterXY:
+    def __init__(
+            self,
+            x: Series,
+            y: Series,
+            xunits: str = None,
+            yunits: str = None,
+            title: str = None,
+            ax: plt.Axes = None,
+            nbins: int = 0,
+            xlim: list = None,
+            ylim: list = None
+    ):
         """
 
+        https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.scatter.html
+
         """
-        self.xy_df = pd.concat([x, y], axis=1)
-        self.xy_df = self.xy_df.dropna()
         self.xname = x.name
         self.yname = y.name
         self.xunits = xunits
         self.yunits = yunits
+        self.ax = ax
+        self.nbins = nbins
+        self.xlim = xlim
+        self.ylim = ylim
 
-        self.title = title if title else None
-
-        # # Create axis
-        # if ax:
-        #     # If ax is given, plot directly to ax, no fig needed
-        #     self.fig = None
-        #     self.ax = ax
-        #     self.showplot = False
-        # else:
-        #     # If no ax is given, create fig and ax and then show the plot
-        #     self.fig, self.ax = pf.create_ax()
-        #     self.showplot = True
+        self.xy_df = pd.concat([x, y], axis=1)
+        self.xy_df = self.xy_df.dropna()
 
-    def plot(self, ax, nbins: int = 10, lw: float or None = theme.WIDTH_LINE_DEFAULT):
+        self.title = title if title else f"{self.yname} vs. {self.xname}"
+
+        self.fig = None
+
+        if self.nbins > 0:
+            self._databinning()
+
+    def _databinning(self):
+        group, bins = pd.qcut(self.xy_df[self.xname], q=self.nbins, retbins=True, duplicates='drop')
+        groupcol = f'GROUP_{self.xname}'
+        self.xy_df[groupcol] = group
+        self.xy_df_binned = self.xy_df.groupby(groupcol).agg({'mean', 'std', 'count'})
+
+    def plot(self):
+        """Generate plot"""
+        if not self.ax:
+            # Create ax if none is given
+            self.fig, self.ax = pf.create_ax(figsize=(8, 8))
+            self._plot()
+            plt.tight_layout()
+            self.fig.show()
+        else:
+            # Otherwise plot to given ax
+            self._plot()
+
+    def _plot(self, nbins: int = 10):
+        """Generate plot on axis"""
         nbins += 1  # To include zero
-
         label = self.yname
-
-        if ax: self.ax=ax
-
-        self.ax.plot(self.xy_df[self.xname],
-                     self.xy_df[self.yname],
-                     color='black', alpha=1,
-                     ls='-', lw=lw,
-                     marker='o', markeredgecolor='none', ms=5,
-                     zorder=99, label=label)
+        self.ax.scatter(x=self.xy_df[self.xname],
+                        y=self.xy_df[self.yname],
+                        c='none',
+                        s=40,
+                        marker='o',
+                        edgecolors='#607D8B',
+                        label=label)
+
+        if self.nbins > 0:
+            _min = self.xy_df_binned[self.yname]['count'].min()
+            _max = self.xy_df_binned[self.yname]['count'].max()
+            self.ax.scatter(x=self.xy_df_binned[self.xname]['mean'],
+                            y=self.xy_df_binned[self.yname]['mean'],
+                            c='none',
+                            s=80,
+                            marker='o',
+                            edgecolors='r',
+                            lw=2,
+                            label=f"binned data, mean±SD "
+                                  f"({_min}-{_max} values per bin)")
+            self.ax.errorbar(x=self.xy_df_binned[self.xname]['mean'],
+                             y=self.xy_df_binned[self.yname]['mean'],
+                             xerr=self.xy_df_binned[self.xname]['std'],
+                             yerr=self.xy_df_binned[self.yname]['std'],
+                             elinewidth=3, ecolor='red', alpha=.6, lw=0)
 
         self._apply_format()
-
         self.ax.locator_params(axis='x', nbins=nbins)
         self.ax.locator_params(axis='y', nbins=nbins)
 
-        # if self.showplot:
-        #     self.fig.show()
-
     def _apply_format(self):
 
-        # ymin = self.series.min()
-        # ymax = self.series.max()
-        # self.ax.set_ylim(ymin, ymax)
+        if self.xlim:
+            xmin = self.xlim[0]
+            xmax = self.xlim[1]
+            self.ax.set_xlim(xmin, xmax)
+
+        if self.ylim:
+            ymin = self.ylim[0]
+            ymax = self.ylim[1]
+            self.ax.set_ylim(ymin, ymax)
 
-        # pf.add_zeroline_y(ax=self.ax, data=self.series)
+        pf.add_zeroline_y(ax=self.ax, data=self.xy_df[self.yname])
 
         pf.default_format(ax=self.ax,
                           ax_xlabel_txt=self.xname,
@@ -90,28 +126,44 @@ def _apply_format(self):
 
 
 def example():
-    pass
-    # # Test data
-    # from diive.core.io.filereader import ReadFileType
-    # loaddatafile = ReadFileType(
-    #     filetype='DIIVE_CSV_30MIN',
-    #     filepath=r"F:\Dropbox\luhk_work\20 - CODING\21 - DIIVE\diive-gui\src\main\resources\base\example_files\ExampleFile_DIIVE_CSV_30T.diive.csv",
-    #     # filepath=r"F:\Dropbox\luhk_work\_current\fp2022\7-14__IRGA627572__addingQCF0\CH-DAV_FP2022.1_1997-2022.08_ID20220826234456_30MIN.diive.csv",
-    #     data_nrows=None)
-    # data_df, metadata_df = loaddatafile.get_filedata()
-    #
-    # series_col = 'co2_flux'
-    # series = data_df[series_col].copy()
-    # series_units = metadata_df.loc[series_col]['UNITS']
-    #
+    from pathlib import Path
+    FOLDER = r"F:\Sync\luhk_work\20 - CODING\21 - DIIVE\diive\notebooks\Workbench\FLUXNET_CH4-N2O_Committee_WP2\data"
+
+    # from diive.core.io.filereader import search_files, MultiDataFileReader
+    # filepaths = search_files(FOLDER, "*.csv")
+    # filepaths = [fp for fp in filepaths if "_fluxnet_" in fp.stem and fp.stem.endswith("_adv")]
+    # print(filepaths)
+    # fr = MultiDataFileReader(filetype='EDDYPRO_FLUXNET_30MIN', filepaths=filepaths)
+    # df = fr.data_df
+    # from diive.core.io.files import save_parquet
+    # save_parquet(outpath=FOLDER, filename="data", data=df)
+
+    from diive.core.io.files import load_parquet
+    filepath = Path(FOLDER) / 'data.parquet'
+    df = load_parquet(filepath=filepath)
+
+    _filter = df['SW_IN_POT'] > 50
+    df = df[_filter].copy()
+
+    # fluxcol = 'FCH4'
+    xcol = 'Rg_1_1_1'
+    ycol = 'Ta_1_1_1'
+
+    x = df[xcol].copy()
+    y = df[ycol].copy()
+
+    # # Plot to given ax
     # fig, ax = pf.create_ax()
-    # # series_units = r'($\mathrm{gC\ m^{-2}}$)'
-    # TimeSeries(ax=ax,
-    #            series=series,
-    #            series_units=series_units).plot()
+    # Scatter(x=x, y=y, ax=ax).plot()
     # fig.tight_layout()
     # fig.show()
 
+    # Plot without given ax
+    ScatterXY(x=x, y=y, nbins=10).plot()
+    # Scatter(x=x, y=y, nbins=10, ylim=[0, 2]).plot()
+
+    # series_units = r'($\mathrm{gC\ m^{-2}}$)'
+
 
 if __name__ == '__main__':
     example()

diive/core/plotting/styles/LightTheme.py

@@ -33,7 +33,7 @@
 TYPE_MARKER_DEFAULT = 'o'
 
 # GENERAL DEFAULTS FOR ALL PLOT TYPES
-LINEWIDTH_SPINES = 2  # Lines framing the plot
+LINEWIDTH_SPINES = 1  # Lines framing the plot
 LINEWIDTH_ZERO = 1
 
 # SCATTER PLOT

diive/pkgs/analyses/quantiles.py

@@ -2,7 +2,7 @@
 import pandas as pd
 from pandas import Series, DataFrame
 
-from diive.core.plotting.scatter import Scatter
+from diive.core.plotting.scatter import ScatterXY
 
 
 def percentiles(series: Series, showplot: bool = True) -> DataFrame:
@@ -18,9 +18,9 @@ def percentiles(series: Series, showplot: bool = True) -> DataFrame:
     print(percentiles_df[percentiles_df["PERCENTILE"].isin(show)])
 
     if showplot:
-        scatter = Scatter(x=percentiles_df['PERCENTILE'],
-                          y=percentiles_df['VALUE'],
-                          title=f"Percentile values: {series.name}")
+        scatter = ScatterXY(x=percentiles_df['PERCENTILE'],
+                            y=percentiles_df['VALUE'],
+                            title=f"Percentile values: {series.name}")
         scatter.plot(nbins=10)
 
     # plt.plot(percentiles_df['PERCENTILE'], percentiles_df['VALUE'],

diive/pkgs/flux/uncertainty.py

@@ -11,7 +11,7 @@
 import diive.core.plotting.styles.LightTheme as theme
 from diive.core.dfun.frames import df_between_two_dates
 from diive.core.plotting.plotfuncs import default_format, default_legend, nice_date_ticks
-from diive.core.plotting.scatter import Scatter
+from diive.core.plotting.scatter import ScatterXY
 
 
 # todo
@@ -177,7 +177,7 @@ def showplot_random_uncertainty(self):
         ax_orig.set_title(f"Measured {self.fluxcol} with METHOD 1 uncertainties")
 
         # Scatter (measured, method 1)
-        Scatter(x=_df[self.fluxcol], y=_df[self.randunccol]).plot(lw=0, ax=ax_scatter)
+        ScatterXY(x=_df[self.fluxcol], y=_df[self.randunccol]).plot(lw=0, ax=ax_scatter)
         ax_scatter.set_title(f"Measured {self.fluxcol} with METHOD 1 uncertainties")
 
         # Time series (gapfilled, method 1-4)

diive/pkgs/flux/ustar_detection.py

@@ -139,36 +139,33 @@ def _calc_swin_pot(self):
 
 def example():
     from pathlib import Path
-    OUTPATH = r"F:\TMP\fru"
+    FOLDER = r"F:\Sync\luhk_work\20 - CODING\21 - DIIVE\diive\notebooks\Workbench\FLUXNET_CH4-N2O_Committee_WP2\data"
 
     # from diive.core.io.filereader import search_files, MultiDataFileReader
-    # FOLDER = r"L:\Sync\luhk_work\CURRENT\fru\Level-1_results_fluxnet_2020-2022"
     # filepaths = search_files(FOLDER, "*.csv")
-    # filepaths = [fp for fp in filepaths
-    #              if "_fluxnet_" in fp.stem
-    #              and fp.stem.endswith("_adv")]
+    # filepaths = [fp for fp in filepaths if "_fluxnet_" in fp.stem and fp.stem.endswith("_adv")]
     # print(filepaths)
     # fr = MultiDataFileReader(filetype='EDDYPRO_FLUXNET_30MIN', filepaths=filepaths)
     # df = fr.data_df
     # from diive.core.io.files import save_parquet
-    # save_parquet(outpath=OUTPATH, filename="data", data=df)
+    # save_parquet(outpath=FOLDER, filename="data", data=df)
 
     from diive.core.io.files import load_parquet
-    filepath = Path(OUTPATH) / 'data.parquet'
+    filepath = Path(FOLDER) / 'data.parquet'
     df = load_parquet(filepath=filepath)
 
-    UstarDetectionMPT(
-        df=df,
-        nee_col='FC',
-        ta_col='TA_1_1_1',
-        ustar_col='USTAR',
-        n_bootstraps=10,
-        swin_pot_col='SW_IN_POT',
-        nighttime_threshold=20,
-        utc_offset=1,
-        lat=47.115833,
-        lon=8.537778
-    )
+    # UstarDetectionMPT(
+    #     df=df,
+    #     nee_col='FC',
+    #     ta_col='TA_1_1_1',
+    #     ustar_col='USTAR',
+    #     n_bootstraps=10,
+    #     swin_pot_col='SW_IN_POT',
+    #     nighttime_threshold=20,
+    #     utc_offset=1,
+    #     lat=47.115833,
+    #     lon=8.537778
+    # )
 
 
 if __name__ == '__main__':