Changes to documentation (multiple pages)

eloualiche · Jul 6, 2023 · 73e856e · 73e856e
1 parent ada1801
commit 73e856e
Show file tree

Hide file tree

Showing 17 changed files with 297 additions and 73 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,7 +3,7 @@
 # STANDARD JULIA IGNORE
 /Manifest.toml
 docs/build/
-
+.DS_Store
 # ---------------------------------------------------------
 
 

diff --git a/docs/.DS_Store b/docs/.DS_Store
diff --git a/docs/build/index.html b/docs/build/index.html
diff --git a/docs/build/search/index.html b/docs/build/search/index.html
@@ -1,2 +1,2 @@
 <!DOCTYPE html>
-<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Search · FinanceRoutines.jl</title><script data-outdated-warner src="../assets/warner.js"></script><link href="https://cdnjs.cloudflare.com/ajax/libs/lato-font/3.0.0/css/lato-font.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/juliamono/0.045/juliamono.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.13.24/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL=".."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../assets/documenter.js"></script><script src="../siteinfo.js"></script><script src="../../versions.js"></script><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-dark.css" data-theme-name="documenter-dark" data-theme-primary-dark/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><div class="docs-package-name"><span class="docs-autofit"><a href="../">FinanceRoutines.jl</a></span></div><form class="docs-search" action><input class="docs-search-query" id="documenter-search-query" name="q" type="text" placeholder="Search docs"/></form><ul class="docs-menu"><li><a class="tocitem" href="../">Home</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><nav class="breadcrumb"><ul class="is-hidden-mobile"><li class="is-active"><a href>Search</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Search</a></li></ul></nav><div class="docs-right"><a class="docs-settings-button fas fa-cog" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-sidebar-button fa fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a></div></header><article><p id="documenter-search-info">Loading search...</p><ul id="documenter-search-results"></ul></article><nav class="docs-footer"><p class="footer-message">Powered by <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> and the <a href="https://julialang.org/">Julia Programming Language</a>.</p></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> version 0.27.25 on <span class="colophon-date" title="Wednesday 5 July 2023 11:31">Wednesday 5 July 2023</span>. Using Julia version 1.9.1.</p></section><footer class="modal-card-foot"></footer></div></div></div></body><script src="../search_index.js"></script><script src="../assets/search.js"></script></html>
+<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Search · FinanceRoutines.jl</title><script data-outdated-warner src="../assets/warner.js"></script><link href="https://cdnjs.cloudflare.com/ajax/libs/lato-font/3.0.0/css/lato-font.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/juliamono/0.045/juliamono.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.13.24/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL=".."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../assets/documenter.js"></script><script src="../siteinfo.js"></script><script src="../../versions.js"></script><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-dark.css" data-theme-name="documenter-dark" data-theme-primary-dark/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><div class="docs-package-name"><span class="docs-autofit"><a href="../">FinanceRoutines.jl</a></span></div><form class="docs-search" action><input class="docs-search-query" id="documenter-search-query" name="q" type="text" placeholder="Search docs"/></form><ul class="docs-menu"><li><a class="tocitem" href="../">Home</a></li><li><span class="tocitem">Manual</span><ul><li><a class="tocitem" href="../man/wrds_guide/">WRDS User Guide</a></li></ul></li><li><span class="tocitem">Demos</span><ul><li><a class="tocitem" href="../demo/beta/">Estimating Beta</a></li></ul></li><li><span class="tocitem">Library</span><ul><li><a class="tocitem" href="../lib/public/">Public Interface</a></li><li><a class="tocitem" href="../lib/internals/">Package Internals</a></li></ul></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><nav class="breadcrumb"><ul class="is-hidden-mobile"><li class="is-active"><a href>Search</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Search</a></li></ul></nav><div class="docs-right"><a class="docs-settings-button fas fa-cog" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-sidebar-button fa fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a></div></header><article><p id="documenter-search-info">Loading search...</p><ul id="documenter-search-results"></ul></article><nav class="docs-footer"><p class="footer-message">Powered by <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> and the <a href="https://julialang.org/">Julia Programming Language</a>.</p></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> version 0.27.25 on <span class="colophon-date" title="Thursday 6 July 2023 14:43">Thursday 6 July 2023</span>. Using Julia version 1.9.1.</p></section><footer class="modal-card-foot"></footer></div></div></div></body><script src="../search_index.js"></script><script src="../assets/search.js"></script></html>
diff --git a/docs/build/search_index.js b/docs/build/search_index.js
diff --git a/docs/make.jl b/docs/make.jl
@@ -1,15 +1,29 @@
 # Inside make.jl
-# push!(LOAD_PATH,"../src/")
+push!(LOAD_PATH, "../src/")
+# push!(LOAD_PATH, "./demo/", "./man/, ")
 
 using FinanceRoutines
 using Documenter
+
 makedocs(
-         sitename = "FinanceRoutines.jl",
-         modules  = [FinanceRoutines],
-         pages=[
-                "Home" => "index.md"
-               ]
-               )
+    format = Documenter.HTML(),
+    sitename = "FinanceRoutines.jl",
+    modules  = [FinanceRoutines],
+    authors = "Erik Loualiche",
+    pages=[
+        "Home" => "index.md",
+        "Manual" => [
+            "man/wrds_guide.md"
+        ],
+        "Demos" => [
+            "demo/beta.md"
+        ],
+        "Library" => [
+            "lib/public.md",
+            "lib/internals.md"
+        ]
+    ]
+)
 
 deploydocs(
     repo="github.com/eloualiche/FinanceRoutines.jl.git",
@@ -21,4 +35,3 @@ deploydocs(;
     target = "build",
     branch = "gh-pages",
 )
-
diff --git a/docs/src/demo/beta.md b/docs/src/demo/beta.md
@@ -0,0 +1 @@
+# Estimating Beta
diff --git a/docs/src/index.md b/docs/src/index.md
@@ -1,12 +1,40 @@
-# FinanceRoutines
+# FinanceRoutines.jl
 
+*Some useful tools to work with academic financial data in Julia*
+
+## Introduction
+
+This package provides a collection of source code *lexers* for various languages and markup
+formats and a selection of *themes* that can be used to customise the style of the formatted
+source code. Additional *lexer* definitions are straightforward to add and are based on the
+regular expression lexing mechanism used by [Pygments](http://pygments.org/).
+
+## Installation
+
+`FinanceRoutines.jl` is a not yet a registered package.
+You can install it from github  via
+
+```julia
+import Pkg
+Pkg.add("https://github.com/eloualiche/FinanceRoutines.jl")
+```
+
+## Usage
+
+  - Using WRDS (CRSP, Compustat, etc)
+      + See the [WRDS User Guide](@ref) for an introduction to using the package to download data from WRDS
+
+  - See the demo to how this integrates into standard estimations in the [Estimating Beta](@ref) demo.
+
+
+
+## Other Resources
 
 ```@contents
 Pages = ["index.md"]
 Depth = 3
 ```
 
-
 ## Functions
 
 ```@docs
@@ -16,8 +44,8 @@ import_FF3
 ```
 
 ```@docs
-build_Funda
-build_MSF
+build_Funda!
+build_MSF!
 ```
 
 ```@docs

diff --git a/docs/src/lib/internals.md b/docs/src/lib/internals.md
@@ -0,0 +1,8 @@
+# Package Internals
+
+## `FinanceRoutines` Module
+
+```@autodocs
+Modules = [FinanceRoutines]
+Public = false
+```
diff --git a/docs/src/lib/public.md b/docs/src/lib/public.md
@@ -0,0 +1,8 @@
+# Public Interface
+
+## `FinanceRoutines` Module
+
+```@autodocs
+Modules = [FinanceRoutines]
+Private = false
+```
diff --git a/docs/src/man/wrds_guide.md b/docs/src/man/wrds_guide.md
@@ -0,0 +1,19 @@
+# WRDS User Guide
+
+```julia
+using FinanceRoutines
+# Set up a wrds connection
+wrds_conn = FinanceRoutines.open_wrds_pg()
+
+# CRSP
+df_msf = import_MSF(wrds_conn); # Import the monthly stock file
+df_msf = build_MSF(df_msf); # Run common processing
+# Compustat
+df_funda = import_Funda(wrds_conn);
+df_funda = build_Funda(df_funda);
+# Merge both files
+df_linktable = FinanceRoutines.import_ccm_link(wrds_conn)
+df_msf = link_MSF(df_linktable, df_msf) # merge gvkey on monthly stock file
+df_msf = innerjoin(df_msf, df_funda, on = [:gvkey, :date_y], matchmissing=:notequal)
+```
+
diff --git a/src/FinanceRoutines.jl b/src/FinanceRoutines.jl
@@ -16,7 +16,7 @@ import IntervalSets:(..)
 import LibPQ: LibPQ.execute, LibPQ.Connection
 import Missings: Missings, missing
 import MonthlyDates: MonthlyDate
-import PanelShift: panellag!
+import PanelShift: panellag!, tlag
 import ShiftedArrays: lag
 import Tables: columntable
 import WeakRefStrings: String3, String7, String15
@@ -27,7 +27,8 @@ import ZipFile: ZipFile.Reader
 # ------------------------------------------------------------------------------------------
 # Import functions
 include("Utilities.jl")
-include("ImportFinanceData.jl")
+include("ImportFamaFrench.jl")
+include("ImportYields.jl")
 include("ImportCRSP.jl")
 include("ImportComp.jl")
 include("Merge_CRSP_Comp.jl")
@@ -38,20 +39,26 @@ include("Merge_CRSP_Comp.jl")
 # List of exported functions
 export greet_FinanceRoutines  # for debugging
 
+# Yields on Treasuries
+export import_GSW
+export estimate_yield_GSW!, estimate_price_GSW!, estimate_return_GSW!
+
+# Fama-French data
+export import_FF3
+
 # WRDS
 # -- CRSP
 export import_MSF             # import Monthly Stock File
 export import_DSF             # import Daily Stock File
-export build_MSF              # clean Monthly Stock File
+export build_MSF!              # clean Monthly Stock File
 # -- Funda
 export import_Funda
-export build_Funda
+export build_Funda!
 # -- Link
 export link_Funda
 export link_MSF
 
-# FF
-export import_FF3
+
 # ------------------------------------------------------------------------------------------
 
 

diff --git a/src/ImportCRSP.jl b/src/ImportCRSP.jl
@@ -54,13 +54,13 @@
 """
     import_MSF(wrds_conn; date_range, variables)
     import_MSF(; 
-        date_range=(Date("1900-01-01"), Date("2030-01-01"), 
+        date_range::Tuple{Date, Date} = (Date("1900-01-01"), Dates.today()),
         variables::String = "", user="", password="")
 
 Import the CRSP Monthly Stock File (MSF) from CRSP on WRDS PostGre server
 
 # Arguments
-- `wrds_conn::Connection`: An existing PostGreSQL connection to WRDS; creates one if empty
+- `wrds_conn::Connection`: An existing Postgres connection to WRDS; creates one if empty
 
 # Keywords
 - `date_range::Tuple{Date, Date}`: A tuple of dates to select data (limits the download size)
@@ -72,7 +72,7 @@ Import the CRSP Monthly Stock File (MSF) from CRSP on WRDS PostGre server
 - `df_msf_final::DataFrame`: DataFrame with msf crsp file
 """
 function import_MSF(wrds_conn::Connection;
-    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Date("2030-01-01")),
+    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Dates.today()),
     variables::String = ""
     )
 
@@ -180,81 +180,83 @@ end
 
 # ------------------------------------------------------------------------------------------
 """
-    build_MSF(df_msf::DataFrame; save)
+    build_MSF!(df_msf::DataFrame; save)
 
-Clean up the compustat funda file download from crsp (see `import_Funda`)
+Clean up the CRSP Monthly Stock File (see `import_MSF`)
 
 # Arguments
-- `df_funda::DataFrame`: A standard dataframe with compustat data (minimum variables are in `import_Funda`)
+- `df::DataFrame`: A standard dataframe with compustat data (minimum variables are in `import_Funda`)
 
 # Keywords
 - `save::String`: Save a gzip version of the data on path `\$save/funda.csv.gz`; Default does not save the data.
-
-# Returns
-- `df_msf::DataFrame`: DataFrame with compustat funda file "cleaned"
+- `trim_col::Bool`: Only keep a subset of relevant columns in the final dataset
 """
-function build_MSF(df_msf::DataFrame;
-    save::String = ""
-    )
+function build_MSF!(df::DataFrame;
+    save::String = "", trim_col::Bool = false)
+
+ # Check that all necessary variables are in
+    ["mktcap", "shrout", "altprc", "permno", "datem", "dlstcd", "ret", "dlret"]
+
 
 # Create marketcap:
-    @rtransform!(df_msf, :mktcap = abs(:shrout * :altprc)) # in 1000s
-    df_msf[ isequal.(df_msf.mktcap, 0), :mktcap] .= missing;
+    @rtransform!(df, :mktcap = abs(:shrout * :altprc)) # in 1000s
+    df[ isequal.(df.mktcap, 0), :mktcap] .= missing;
 
 # Lagged marketcap
-    sort!(df_msf, [:permno, :datem])
+    sort!(df, [:permno, :datem])
     # method 1: lag and then merge back
     # df_msf_mktcap_lag = @select(df_msf,
     #         :datem = :datem + Month(1), :permno, :l1m_mktcap2 = :mktcap)
     # df_msf = leftjoin(df_msf, df_msf_mktcap_lag, on = [:permno, :datem])
-    panellag!(df_msf, :permno, :datem, 
+    panellag!(df, :permno, :datem, 
         :mktcap, :l1m_mktcap, Month(1))
 
 # Adjusted returns (see tidy finance following Bali, Engle, and Murray)
-    @rtransform! df_msf :ret_adj = 
+    @rtransform! df :ret_adj = 
         ismissing(:dlstcd) ? :ret : 
             !ismissing(:dlret) ? :dlret :
                 (:dlstcd ∈ (500, 520, 580, 584)) || ((:dlstcd >= 551) & (:dlstcd <= 574)) ? -0.3 :
                     :dlstcd == 100 ? :ret : -1.0
 
 # select variables and save
-    select!(df_msf, :permno, :date, :ret, :mktcap, :l1m_mktcap, :retx, 
-        :naics, :hsiccd)
+    if trim_col
+        select!(df, :permno, :date, :ret, :mktcap, :l1m_mktcap, :retx, :naics, :hsiccd)
+    end
     if !(save == "")
-        CSV.write(save * "/msf.csv.gz", df_msf, compress=true)
+        CSV.write(save * "/msf.csv.gz", df, compress=true)
     end
 
-    return df_msf
+    return df
 end
 
 
 function build_MSF(wrds_conn::Connection;
-    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Date("2030-01-01")),
+    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Dates.today()),
     save::Bool = false,
     )
 
-    df_msf = import_MSF(wrds_conn; date_range=date_range);
-    df_msf = build_msf(df_msf, save = save)
-    return df_msf
+    df = import_MSF(wrds_conn; date_range=date_range);
+    df = build_msf(df, save = save)
+    return df
 end
 
 
 function build_MSF(;
-    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Date("2030-01-01")),
+    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Dates.today()),
     save::Bool = false,
     )
 
-    df_msf = import_MSF(;date_range);
-    df_msf = build_msf(df_msf, save = save)
+    df = import_MSF(;date_range);
+    df = build_msf(df, save = save)
 
-    return df_msf
+    return df
 end
 # ------------------------------------------------------------------------------------------
 
 
 # ------------------------------------------------------------------------------------------
 function import_DSF(wrds_conn::Connection;
-    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Date("2030-01-01")),
+    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Dates.today()),
     variables::String
     )
 
@@ -276,7 +278,7 @@ end
 
 # when there are no connections establisheds
 function import_DSF(;
-    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Date("2030-01-01")),
+    date_range::Tuple{Date, Date} = (Date("1900-01-01"), Dates.today()),
     variables::String = "",
     user::String = "", password::String = "")