Merge pull request #1 from niaid/dev

v0.0.3
niaid · Apr 25, 2022 · 23afa5e · 23afa5e
2 parents b8e99c2 + d9ece33
commit 23afa5e
Show file tree

Hide file tree

Showing 10 changed files with 198 additions and 87 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,16 +8,16 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   # Python
-  # - repo: https://github.com/PyCQA/flake8
-  #   rev: 4.0.1
-  #   hooks:
-  #     - id: flake8
-  #       args: [
-  #           "--ignore",
-  #           "E203,W503,E741", # ignore these rules, to comply with black. See: https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated
-  #           "--max-line-length",
-  #           "88", # 90-ish is a good choice. See: https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length
-  #         ]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: [
+            "--ignore",
+            "E203,W503,E741", # ignore these rules, to comply with black. See: https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated
+            "--max-line-length",
+            "88", # 90-ish is a good choice. See: https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length
+          ]
   - repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:

diff --git a/cloudwatcher/_version.py b/cloudwatcher/_version.py
@@ -1 +1 @@
-__version__ = "0.0.2"
+__version__ = "0.0.3"
diff --git a/cloudwatcher/argparser.py b/cloudwatcher/argparser.py
@@ -31,9 +31,7 @@ def build_argparser():
     region = "us-east-1"
 
     # add argument parser
-    parser = _VersionInHelpParser(
-        description="CloudWatch metrics explorer. In order to use the tool a CloudWatchAgent process must be running on the EC2 instance to be monitored."
-    )
+    parser = _VersionInHelpParser(description="CloudWatch logs and metrics explorer.")
 
     subparsers = parser.add_subparsers(dest="command")
 

diff --git a/cloudwatcher/cli.py b/cloudwatcher/cli.py
@@ -123,6 +123,6 @@ def main():
         if args.save:
             log_watcher.save_log_file(
                 file_path=os.path.join(
-                    args.dir, f"{args.log_grop_name}-{args.log_stream_name}.log"
+                    args.dir, f"{args.log_group_name}-{args.log_stream_name}.log"
                 )
             )
diff --git a/cloudwatcher/logwatcher.py b/cloudwatcher/logwatcher.py
@@ -188,3 +188,16 @@ def _datestr(timestamp: int, fmt_str: str = "%d-%m-%Y %H:%M:%S") -> str:
                 fmt_str.format(time=_datestr(e["timestamp"]), message=msg.strip())
             )
         return formatted_log_list
+
+    def save_log_file(self, file_path: str) -> None:
+        """
+        Save the log file to the specified path
+
+        :param str log_file_path: The path to save the log file
+        """
+        logs, _ = self.return_formatted_logs()
+        with open(file_path, "w") as f:
+            f.write(logs)
+        _LOGGER.info(
+            f"Logs '{self.log_group_name}/{self.log_stream_name}' saved to: {file_path}"
+        )
diff --git a/docs/API_documentation.md b/docs/API_documentation.md
@@ -7,7 +7,7 @@ document.addEventListener('DOMContentLoaded', (event) => {
 </script>
 
 <style>
-h3 .content {
+h3 .content { 
     padding-left: 22px;
     text-indent: -15px;
  }
@@ -18,7 +18,7 @@ h3 .hljs .content {
     martin-bottom: 0px;
 }
 h4 .content, table .content, p .content, li .content { margin-left: 30px; }
-h4 .content {
+h4 .content { 
     font-style: italic;
     font-size: 1em;
     margin-bottom: 0px;
@@ -34,7 +34,7 @@ A base class for CloudWatch managers
 
 
 ```python
-def __init__(self, service_name: str, region_name: Union[str, NoneType]=None, aws_access_key_id: Union[str, NoneType]=None, aws_secret_access_key: Union[str, NoneType]=None, aws_session_token: Union[str, NoneType]=None, log_debug: Union[bool, NoneType]=False) -> None
+def __init__(self, service_name: str, aws_region_name: Union[str, NoneType]=None, aws_access_key_id: Union[str, NoneType]=None, aws_secret_access_key: Union[str, NoneType]=None, aws_session_token: Union[str, NoneType]=None) -> None
 ```
 
 Initialize CloudWatcher
@@ -45,7 +45,6 @@ Initialize CloudWatcher
 - `aws_access_key_id` (`Optional[str]`):  The AWS access key ID
 - `aws_secret_access_key` (`Optional[str]`):  The AWS secret access key
 - `aws_session_token` (`Optional[str]`):  The AWS session token
-- `log_debug` (`Optional[bool]`):  Whether to log debug messages
 
 
 
@@ -55,7 +54,7 @@ A class for AWS CloudWatch metric retrieval and parsing
 
 
 ```python
-def __init__(self, namespace: str, ec2_instance_id: str, metric_name: str, metric_id: str, metric_unit: str, aws_access_key_id: Union[str, NoneType]=None, aws_secret_access_key: Union[str, NoneType]=None, aws_session_token: Union[str, NoneType]=None, region_name: Union[str, NoneType]=None) -> None
+def __init__(self, namespace: str, ec2_instance_id: str, metric_name: str, metric_id: str, metric_unit: str, aws_access_key_id: Union[str, NoneType]=None, aws_secret_access_key: Union[str, NoneType]=None, aws_session_token: Union[str, NoneType]=None, aws_region_name: Union[str, NoneType]=None) -> None
 ```
 
 Initialize MetricWatcher
@@ -227,7 +226,7 @@ A class for AWS CloudWatch log events retrieval and parsing
 
 
 ```python
-def __init__(self, log_group_name: str, log_stream_name: str, start_token: Union[str, NoneType]=None, aws_access_key_id: Union[str, NoneType]=None, aws_secret_access_key: Union[str, NoneType]=None, aws_session_token: Union[str, NoneType]=None, region_name: Union[str, NoneType]=None) -> None
+def __init__(self, log_group_name: str, log_stream_name: str, start_token: Union[str, NoneType]=None, aws_access_key_id: Union[str, NoneType]=None, aws_secret_access_key: Union[str, NoneType]=None, aws_session_token: Union[str, NoneType]=None, aws_region_name: Union[str, NoneType]=None) -> None
 ```
 
 Initialize LogWatcher
@@ -290,6 +289,18 @@ A generator that yields formatted log events
 
 
 
+```python
+def save_log_file(self, file_path: str) -> None
+```
+
+Save the log file to the specified path
+#### Parameters:
+
+- `log_file_path` (`str`):  The path to save the log file
+
+
+
+
 ```python
 def stream_cloudwatch_logs(self, events_limit: int=1000, max_retry_attempts: int=5) -> List[Dict[str, str]]
 ```
@@ -372,4 +383,4 @@ Class to establish the interface for a timed metric handling
 
 
 
-*Version Information: `cloudwatcher` v0.0.2, generated by `lucidoc` v0.4.3*
+*Version Information: `cloudwatcher` v0.0.3, generated by `lucidoc` v0.4.3*
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -0,0 +1,15 @@
+# Changelog
+
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
+
+## [0.0.3] - 2022-04-25
+
+### Added
+
+- Added support for log saving to file: `cloudwatcher log --save`
+
+## [0.0.2] - 2022-04-25
+
+### Added
+
+- Initial release
diff --git a/docs/usage.md b/docs/usage.md
@@ -1,73 +1,107 @@
+There are two modes of operation on the CLI:
+
+- [`cloudwatcher metric`](#cloudwatch-metrics-monitoring)
+- [`cloudwatcher log`](#cloudwatch-logs-monitoring)
+
+```
+cloudwatcher --help
+```
+
+```
+version: 0.0.2
+usage: cloudwatcher [-h] {metric,log} ...
+
+CloudWatch logs and metrics explorer.
+
+positional arguments:
+  {metric,log}
+    metric      Interact with AWS CloudWatch metrics.
+    log         Interact with AWS CloudWatch logs.
+
+optional arguments:
+  -h, --help    show this help message and exit
+```
+
+## CloudWatch metrics monitoring
+
 The tool is highly configurable and can be used in a variety of ways. Naturally, the [metrics available to be monitored](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/metrics-collected-by-CloudWatch-agent.html) depend on the configuration of the CloudWatchAgent process.
 
 By default the tool will report the `mem_used` metric starting 24 hours ago until present with granularity/period of 1 minute, expressed in Bytes.
 
 Please refer to the usage below for more options:
 
 ```
-usage: cloudwatcher [-h] [-jid JOB_ID] [-q QUERY_JSON] [-i ID] [-m METRIC] -iid
-                     INSTANCE_ID [--uptime] [--days DAYS] [-hr HOURS] [-mi MINUTES]
-                     [-u UNIT] [-s STAT] [-p PERIOD] [--save] [--plot] [-d DIR]
-                     [--debug] [--namespace NAMESPACE]
+clouwatcher metric --help
+```
+
+```
+version: 0.0.2
+usage: cloudwatcher metric [-h] [--version] [--debug] [--aws-region AWS_REGION]
+                           [--aws-access-key-id AWS_ACCESS_KEY_ID]
+                           [--aws-secret-access-key AWS_SECRET_ACCESS_KEY]
+                           [--aws-session-token AWS_SESSION_TOKEN] [--save] [-d DIR]
+                           [-q QUERY_JSON] [-i ID] [-m METRIC] -iid INSTANCE_ID [--uptime]
+                           [--days DAYS] [-hr HOURS] [-mi MINUTES] [-u UNIT] [-s STAT]
+                           [-p PERIOD] [--plot] --namespace NAMESPACE
 
-CloudWatch metrics explorer. In order to use the tool a CloudWatchAgent process must
-be running on the EC2 instance to be monitored.
+Interact with AWS CloudWatch metrics.
 
 optional arguments:
-  -h, --help            show this help message and exit
-  -jid JOB_ID, --job-id JOB_ID
-                        Nephele job ID to use for the EC2 instance ID lookup. This
-                        is not implemented yet.
+  -h, --help                            show this help message and exit
+  --version                             Print version and exit
+  --debug                               Whether debug mode should be launched (default:
+                                        False)
+  --aws-region AWS_REGION               Region to monitor the metrics within. (default:
+                                        us-east-1)
+  --aws-access-key-id AWS_ACCESS_KEY_ID
+                                        AWS Access Key ID to use for authentication
+  --aws-secret-access-key AWS_SECRET_ACCESS_KEY
+                                        AWS Secret Access Key to use for authentication
+  --aws-session-token AWS_SESSION_TOKEN
+                                        AWS Session Token to use for authentication
+  --save                                Whether to save the results to files in the
+                                        selected directory (default: False)
+  -d DIR, --dir DIR                     Directory to store the results in. Used with
+                                        `--save` (default: ./)
   -q QUERY_JSON, --query-json QUERY_JSON
-                        Path to a query JSON file. This is not implemented yet.
-  -i ID, --id ID        The unique identifier to assign to the metric data. Must be
-                        of the form '^[a-z][a-zA-Z0-9_]*$'.
+                                        Path to a query JSON file. This is not implemented
+                                        yet.
+  -i ID, --id ID                        The unique identifier to assign to the metric
+                                        data. Must be of the form '^[a-z][a-zA-Z0-9_]*$'.
+  -m METRIC, --metric METRIC            Name of the metric collected by CloudWatchAgent
+                                        (default: mem_used)
   -iid INSTANCE_ID, --instance-id INSTANCE_ID
-                        Instance ID, needs to follow 'i-<numbers>' format
-  --uptime              Display the uptime of the instance in seconds. It's either
-                        calculated precisely if the instance is still running, or
-                        estimated based on the reported metrics.
-
- Options for metric collection start time:
-  --days DAYS           How many days to subtract from the current date to determine
-                        the metric collection start time (default: 1). Uptime will
-                        be estimated in the timespan starting at least 15 ago.
-  -hr HOURS, --hours HOURS
-                        How many hours to subtract from the current time to
-                        determine the metric collection start time (default: 0).
-                        Uptime will be estimated in the timespan starting at least
-                        15 ago.
-  -mi MINUTES, --minutes MINUTES
-                        How many minutes to subtract from the current time to
-                        determine the metric collection start time (default: 0).
-                        Uptime will be estimated in the timespan starting at least
-                        15 ago.
-
- Options for metric measurement:
-   --namespace NAMESPACE
-                        Namespace to monitor the metrics within. This value must
-                        match the 'Namespace' value in the config (default:
-                        NepheleNamespace)
-  -m METRIC, --metric METRIC
-                        Name of the metric collected by CloudWatchAgent (default:
-                        mem_used)
-  -u UNIT, --unit UNIT  If you omit Unit then all data that was collected with any
-                        unit is returned. If you specify a unit, it acts as a filter
-                        and returns only data that was collected with that unit
-                        specified. Use 'Bytes' for memory (default: Bytes)
-  -s STAT, --stat STAT  The statistic to apply over the time intervals, e.g.
-                        'Maximum' (default: Maximum)
-  -p PERIOD, --period PERIOD
-                        The granularity, in seconds, of the returned data points.
-                        Choices: 1, 5, 10, 30, 60, or any multiple of 60 (default:
-                        60)
-  Output options:
-  --save                Whether to store the response and metric data in JSON and
-                        CSV files (default: False)
-  --plot                Whether to plot the metric data (default: False)
-  -d DIR, --dir DIR     Directory to store the results in. Used with `--save`
-                        (default: ./)
-  --debug               Whether debug mode should be launched (default: False)
+                                        Instance ID, needs to follow 'i-<numbers>' format
+  --uptime                              Display the uptime of the instance in seconds.
+                                        It's either calculated precisely if the instance
+                                        is still running, or estimated based on the
+                                        reported metrics.
+  --days DAYS                           How many days to subtract from the current date to
+                                        determine the metric collection start time
+                                        (default: 1). Uptime will be estimated in the
+                                        timespan starting at least 15 ago.
+  -hr HOURS, --hours HOURS              How many hours to subtract from the current time
+                                        to determine the metric collection start time
+                                        (default: 0). Uptime will be estimated in the
+                                        timespan starting at least 15 ago.
+  -mi MINUTES, --minutes MINUTES        How many minutes to subtract from the current time
+                                        to determine the metric collection start time
+                                        (default: 0). Uptime will be estimated in the
+                                        timespan starting at least 15 ago.
+  -u UNIT, --unit UNIT                  If you omit Unit then all data that was collected
+                                        with any unit is returned. If you specify a unit,
+                                        it acts as a filter and returns only data that was
+                                        collected with that unit specified. Use 'Bytes'
+                                        for memory (default: Bytes)
+  -s STAT, --stat STAT                  The statistic to apply over the time intervals,
+                                        e.g. 'Maximum' (default: Maximum)
+  -p PERIOD, --period PERIOD            The granularity, in seconds, of the returned data
+                                        points. Choices: 1, 5, 10, 30, 60, or any multiple
+                                        of 60 (default: 60)
+  --plot                                Whether to plot the metric data (default: False)
+  --namespace NAMESPACE                 Namespace to monitor the metrics within. This
+                                        value must match the 'Namespace' value in the
+                                        CloudWatchAgent config (default: None)
 ```
 
 ### Minimal command
@@ -86,3 +120,42 @@ Amazon CloudWatch retains metric data as follows:
 - Data points with a period of 3600 seconds (1 hour) are available for 455 days (15 months).
 
 Select your period of interest accordingly.
+
+## CloudWatch logs monitoring
+
+```
+clouwatcher log --help
+```
+
+```
+version: 0.0.2
+usage: cloudwatcher log [-h] [--version] [--debug] [--aws-region AWS_REGION]
+                        [--aws-access-key-id AWS_ACCESS_KEY_ID]
+                        [--aws-secret-access-key AWS_SECRET_ACCESS_KEY]
+                        [--aws-session-token AWS_SESSION_TOKEN] [--save] [-d DIR] -g
+                        LOG_GROUP_NAME -s LOG_STREAM_NAME
+
+Interact with AWS CloudWatch logs.
+
+optional arguments:
+  -h, --help                            show this help message and exit
+  --version                             Print version and exit
+  --debug                               Whether debug mode should be launched (default:
+                                        False)
+  --aws-region AWS_REGION               Region to monitor the metrics within. (default:
+                                        us-east-1)
+  --aws-access-key-id AWS_ACCESS_KEY_ID
+                                        AWS Access Key ID to use for authentication
+  --aws-secret-access-key AWS_SECRET_ACCESS_KEY
+                                        AWS Secret Access Key to use for authentication
+  --aws-session-token AWS_SESSION_TOKEN
+                                        AWS Session Token to use for authentication
+  --save                                Whether to save the results to files in the
+                                        selected directory (default: False)
+  -d DIR, --dir DIR                     Directory to store the results in. Used with
+                                        `--save` (default: ./)
+  -g LOG_GROUP_NAME, --log-group-name LOG_GROUP_NAME
+                                        The log group name to monitor
+  -s LOG_STREAM_NAME, --log-stream-name LOG_STREAM_NAME
+                                        The log stream name to monitor
+```
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -28,12 +28,13 @@ markdown_extensions:
   - pymdownx.superfences
 nav:
   - Installation & Setup:
-    - Installation: installation.md
-    - EC2 intance setup: EC2_instance_setup.md
-    - AWS credentials: login_credentials.md
+      - Installation: installation.md
+      - EC2 intance setup: EC2_instance_setup.md
+      - AWS credentials: login_credentials.md
   - Usage & Features:
-    - Usage: usage.md
-    - Features: features.md
+      - Usage: usage.md
+      - Features: features.md
   - API documentation: API_documentation.md
   - Development:
-    - Docs development: docs_development.md
+      - Docs development: docs_development.md
+  - Release notes: changelog.md
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cloudwatcher"
-version = "0.0.2"
+version = "0.0.3"
 description = "A tool for monitoring AWS CloudWatch metrics"
 authors = ["Michal Stolarczyk <stolarczyk.michal93@gmail.com>"]