From 85178ee7e5c8c3d802ae117bdfc30f27cb0b5927 Mon Sep 17 00:00:00 2001 From: Joellensilva Date: Wed, 14 Jun 2023 16:48:47 -0300 Subject: [PATCH 1/2] =?UTF-8?q?retornando=20a=20data=20da=20=C3=BAltima=20?= =?UTF-8?q?coleta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/monthlyInfo.go | 17 +++++++++-------- repo/database/dto/annuaISummary.go | 26 +++++++++++++++++++------- repo/database/postgres.go | 5 +++-- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/models/monthlyInfo.go b/models/monthlyInfo.go index 9f38f31..95afaa8 100644 --- a/models/monthlyInfo.go +++ b/models/monthlyInfo.go @@ -77,14 +77,15 @@ type GeneralMonthlyInfo struct { } type AnnualSummary struct { - Year int `json:"year,omitempty"` // Year of the data - AverageCount int `json:"average_count,omitempty"` // Average number of employees - TotalCount int `json:"total_count,omitempty"` // Total number of employees - BaseRemuneration float64 `json:"base_remuneration,omitempty"` // Statistics (Max, Min, Median, Total) - OtherRemunerations float64 `json:"other_remunerations,omitempty"` // Statistics (Max, Min, Median, Total) - Discounts float64 `json:"discounts,omitempty"` // Statistics (Max, Min, Median, Total) - NumMonthsWithData int `json:"months_with_data,omitempty"` - Package *Backup `json:"package,omitempty"` + Year int `json:"year,omitempty"` // Year of the data + AverageCount int `json:"average_count,omitempty"` // Average number of employees + TotalCount int `json:"total_count,omitempty"` // Total number of employees + BaseRemuneration float64 `json:"base_remuneration,omitempty"` // Statistics (Max, Min, Median, Total) + OtherRemunerations float64 `json:"other_remunerations,omitempty"` // Statistics (Max, Min, Median, Total) + Discounts float64 `json:"discounts,omitempty"` // Statistics (Max, Min, Median, Total) + NumMonthsWithData int `json:"months_with_data,omitempty"` + Package *Backup `json:"package,omitempty"` + Timestamp *timestamppb.Timestamp `json:"timestamp,omitempty"` } type RemmunerationSummary struct { diff --git a/repo/database/dto/annuaISummary.go b/repo/database/dto/annuaISummary.go index 3492873..b4b8e55 100644 --- a/repo/database/dto/annuaISummary.go +++ b/repo/database/dto/annuaISummary.go @@ -1,20 +1,30 @@ package dto import ( + "time" + "github.com/dadosjusbr/storage/models" + "google.golang.org/protobuf/types/known/timestamppb" ) type AnnualSummaryDTO struct { - Year int `gorm:"column:ano"` - AverageCount int `gorm:"column:media_num_membros"` - TotalCount int `gorm:"column:total_num_membros"` - BaseRemuneration float64 `gorm:"column:remuneracao_base"` - OtherRemunerations float64 `gorm:"column:outras_remuneracoes"` - Discounts float64 `gorm:"column:descontos"` - NumMonthsWithData int `gorm:"column:meses_com_dados"` + Year int `gorm:"column:ano"` + AverageCount int `gorm:"column:media_num_membros"` + TotalCount int `gorm:"column:total_num_membros"` + BaseRemuneration float64 `gorm:"column:remuneracao_base"` + OtherRemunerations float64 `gorm:"column:outras_remuneracoes"` + Discounts float64 `gorm:"column:descontos"` + NumMonthsWithData int `gorm:"column:meses_com_dados"` + Timestamp time.Time `gorm:"column:timestamp"` } func NewAnnualSummaryDTO(ami models.AnnualSummary) *AnnualSummaryDTO { + var timestamp time.Time + if ami.Timestamp != nil { + timestamp = time.Unix(ami.Timestamp.Seconds, int64(ami.Timestamp.Nanos)) + } else { + timestamp = time.Now() + } return &AnnualSummaryDTO{ Year: ami.Year, AverageCount: ami.AverageCount, @@ -23,6 +33,7 @@ func NewAnnualSummaryDTO(ami models.AnnualSummary) *AnnualSummaryDTO { OtherRemunerations: ami.OtherRemunerations, Discounts: ami.Discounts, NumMonthsWithData: ami.NumMonthsWithData, + Timestamp: timestamp, } } @@ -35,5 +46,6 @@ func (ami *AnnualSummaryDTO) ConvertToModel() *models.AnnualSummary { OtherRemunerations: ami.OtherRemunerations, Discounts: ami.Discounts, NumMonthsWithData: ami.NumMonthsWithData, + Timestamp: timestamppb.New(ami.Timestamp), } } diff --git a/repo/database/postgres.go b/repo/database/postgres.go index c5abfdb..00f25be 100644 --- a/repo/database/postgres.go +++ b/repo/database/postgres.go @@ -284,9 +284,10 @@ func (p *PostgresDB) GetAnnualSummary(agency string) ([]models.AnnualSummary, er SUM(CAST(sumario -> 'remuneracao_base' ->> 'total' AS DECIMAL)) AS remuneracao_base, SUM(CAST(sumario -> 'outras_remuneracoes' ->> 'total' AS DECIMAL)) AS outras_remuneracoes, SUM(CAST(sumario -> 'descontos' ->> 'total' AS DECIMAL)) AS descontos, - COUNT(*) AS meses_com_dados` + COUNT(*) FILTER (WHERE (procinfo::text = 'null' OR procinfo IS NULL)) AS meses_com_dados, + max(timestamp) as timestamp` m := p.db.Model(&dtoAgmi).Select(query) - m = m.Where("id_orgao = ? AND atual = TRUE AND (procinfo::text = 'null' OR procinfo IS NULL) ", agency) + m = m.Where("id_orgao = ? AND atual = TRUE ", agency) m = m.Group("ano, id_orgao").Order("ano ASC") if err := m.Scan(&dtoAmis).Error; err != nil { return nil, fmt.Errorf("error getting annual monthly info: %q", err) From f9242360b94421ab106feccb710b42126438aa1f Mon Sep 17 00:00:00 2001 From: Joellensilva Date: Thu, 15 Jun 2023 14:55:50 -0300 Subject: [PATCH 2/2] retornando data da ultima coleta --- models/agency.go | 21 ++++++----- models/monthlyInfo.go | 17 ++++----- repo/database/dto/agencyDTO.go | 59 ++++++++++++++++++------------ repo/database/dto/annuaISummary.go | 26 ++++--------- repo/database/postgres.go | 9 +++-- 5 files changed, 67 insertions(+), 65 deletions(-) diff --git a/models/agency.go b/models/agency.go index 3c115d8..e4266b4 100644 --- a/models/agency.go +++ b/models/agency.go @@ -1,16 +1,19 @@ package models +import "google.golang.org/protobuf/types/known/timestamppb" + // Agency A Struct containing the main descriptions of each Agency. type Agency struct { - ID string `json:"aid,omitempty"` // 'trt13' - Name string `json:"name,omitempty"` // 'Tribunal Regional do Trabalho 13° Região' - Type string `json:"type,omitempty"` // "R" for Regional, "M" for Municipal, "F" for Federal, "E" for State. - Entity string `json:"entity,omitempty"` // "J" For Judiciário, "M" for Ministério Público, "P" for Procuradorias and "D" for Defensorias. - UF string `json:"uf,omitempty"` // Short code for federative unity. - URL string `json:"url,omitempty"` // Link for state url - Collecting []Collecting `json:"collecting,omitempty"` - TwitterHandle string `json:"twitter_handle,omitempty"` // Agency's twitter handle - OmbudsmanURL string `json:"ombudsman_url,omitempty"` //Agencys's ombudsman url + ID string `json:"aid,omitempty"` // 'trt13' + Name string `json:"name,omitempty"` // 'Tribunal Regional do Trabalho 13° Região' + Type string `json:"type,omitempty"` // "R" for Regional, "M" for Municipal, "F" for Federal, "E" for State. + Entity string `json:"entity,omitempty"` // "J" For Judiciário, "M" for Ministério Público, "P" for Procuradorias and "D" for Defensorias. + UF string `json:"uf,omitempty"` // Short code for federative unity. + URL string `json:"url,omitempty"` // Link for state url + Collecting []Collecting `json:"collecting,omitempty"` + TwitterHandle string `json:"twitter_handle,omitempty"` // Agency's twitter handle + OmbudsmanURL string `json:"ombudsman_url,omitempty"` //Agencys's ombudsman url + LastCollection *timestamppb.Timestamp `json:"last_collection,omitempty"` } // Collecting A Struct containing the day we checked the status of the data and the reasons why we didn't collected it. diff --git a/models/monthlyInfo.go b/models/monthlyInfo.go index 95afaa8..9f38f31 100644 --- a/models/monthlyInfo.go +++ b/models/monthlyInfo.go @@ -77,15 +77,14 @@ type GeneralMonthlyInfo struct { } type AnnualSummary struct { - Year int `json:"year,omitempty"` // Year of the data - AverageCount int `json:"average_count,omitempty"` // Average number of employees - TotalCount int `json:"total_count,omitempty"` // Total number of employees - BaseRemuneration float64 `json:"base_remuneration,omitempty"` // Statistics (Max, Min, Median, Total) - OtherRemunerations float64 `json:"other_remunerations,omitempty"` // Statistics (Max, Min, Median, Total) - Discounts float64 `json:"discounts,omitempty"` // Statistics (Max, Min, Median, Total) - NumMonthsWithData int `json:"months_with_data,omitempty"` - Package *Backup `json:"package,omitempty"` - Timestamp *timestamppb.Timestamp `json:"timestamp,omitempty"` + Year int `json:"year,omitempty"` // Year of the data + AverageCount int `json:"average_count,omitempty"` // Average number of employees + TotalCount int `json:"total_count,omitempty"` // Total number of employees + BaseRemuneration float64 `json:"base_remuneration,omitempty"` // Statistics (Max, Min, Median, Total) + OtherRemunerations float64 `json:"other_remunerations,omitempty"` // Statistics (Max, Min, Median, Total) + Discounts float64 `json:"discounts,omitempty"` // Statistics (Max, Min, Median, Total) + NumMonthsWithData int `json:"months_with_data,omitempty"` + Package *Backup `json:"package,omitempty"` } type RemmunerationSummary struct { diff --git a/repo/database/dto/agencyDTO.go b/repo/database/dto/agencyDTO.go index 61d8a99..34820c1 100644 --- a/repo/database/dto/agencyDTO.go +++ b/repo/database/dto/agencyDTO.go @@ -3,21 +3,24 @@ package dto import ( "encoding/json" "fmt" + "time" "github.com/dadosjusbr/storage/models" + "google.golang.org/protobuf/types/known/timestamppb" "gorm.io/datatypes" ) // Agency A Struct containing the main descriptions of each Agency. type AgencyDTO struct { - ID string `gorm:"column:id"` - Name string `gorm:"column:nome"` - Type string `gorm:"column:jurisdicao"` - Entity string `gorm:"column:entidade"` - UF string `gorm:"column:uf"` - Collecting datatypes.JSON `gorm:"column:coletando"` - TwitterHandle string `gorm:"column:twitter_handle"` - OmbudsmanURL string `gorm:"column:ouvidoria"` + ID string `gorm:"column:id"` + Name string `gorm:"column:nome"` + Type string `gorm:"column:jurisdicao"` + Entity string `gorm:"column:entidade"` + UF string `gorm:"column:uf"` + Collecting datatypes.JSON `gorm:"column:coletando"` + TwitterHandle string `gorm:"column:twitter_handle"` + OmbudsmanURL string `gorm:"column:ouvidoria"` + LastCollection time.Time `gorm:"column:ultima_coleta"` } func (AgencyDTO) TableName() string { @@ -35,14 +38,15 @@ func (a AgencyDTO) ConvertToModel() (*models.Agency, error) { return nil, fmt.Errorf("error while unmarshaling collecting: %q", err) } return &models.Agency{ - ID: a.ID, - Name: a.Name, - Type: a.Type, - Entity: a.Entity, - UF: a.UF, - Collecting: collecting, - TwitterHandle: a.TwitterHandle, - OmbudsmanURL: a.OmbudsmanURL, + ID: a.ID, + Name: a.Name, + Type: a.Type, + Entity: a.Entity, + UF: a.UF, + Collecting: collecting, + TwitterHandle: a.TwitterHandle, + OmbudsmanURL: a.OmbudsmanURL, + LastCollection: timestamppb.New(a.LastCollection), }, nil } @@ -51,14 +55,21 @@ func NewAgencyDTO(agency models.Agency) (*AgencyDTO, error) { if err != nil { return nil, fmt.Errorf("error while marshaling collecting: %q", err) } + var lastCollection time.Time + if agency.LastCollection != nil { + lastCollection = time.Unix(agency.LastCollection.Seconds, int64(agency.LastCollection.Nanos)) + } else { + lastCollection = time.Now() + } return &AgencyDTO{ - ID: agency.ID, - Name: agency.Name, - Type: agency.Type, - Entity: agency.Entity, - UF: agency.UF, - Collecting: collecting, - TwitterHandle: agency.TwitterHandle, - OmbudsmanURL: agency.OmbudsmanURL, + ID: agency.ID, + Name: agency.Name, + Type: agency.Type, + Entity: agency.Entity, + UF: agency.UF, + Collecting: collecting, + TwitterHandle: agency.TwitterHandle, + OmbudsmanURL: agency.OmbudsmanURL, + LastCollection: lastCollection, }, nil } diff --git a/repo/database/dto/annuaISummary.go b/repo/database/dto/annuaISummary.go index b4b8e55..3492873 100644 --- a/repo/database/dto/annuaISummary.go +++ b/repo/database/dto/annuaISummary.go @@ -1,30 +1,20 @@ package dto import ( - "time" - "github.com/dadosjusbr/storage/models" - "google.golang.org/protobuf/types/known/timestamppb" ) type AnnualSummaryDTO struct { - Year int `gorm:"column:ano"` - AverageCount int `gorm:"column:media_num_membros"` - TotalCount int `gorm:"column:total_num_membros"` - BaseRemuneration float64 `gorm:"column:remuneracao_base"` - OtherRemunerations float64 `gorm:"column:outras_remuneracoes"` - Discounts float64 `gorm:"column:descontos"` - NumMonthsWithData int `gorm:"column:meses_com_dados"` - Timestamp time.Time `gorm:"column:timestamp"` + Year int `gorm:"column:ano"` + AverageCount int `gorm:"column:media_num_membros"` + TotalCount int `gorm:"column:total_num_membros"` + BaseRemuneration float64 `gorm:"column:remuneracao_base"` + OtherRemunerations float64 `gorm:"column:outras_remuneracoes"` + Discounts float64 `gorm:"column:descontos"` + NumMonthsWithData int `gorm:"column:meses_com_dados"` } func NewAnnualSummaryDTO(ami models.AnnualSummary) *AnnualSummaryDTO { - var timestamp time.Time - if ami.Timestamp != nil { - timestamp = time.Unix(ami.Timestamp.Seconds, int64(ami.Timestamp.Nanos)) - } else { - timestamp = time.Now() - } return &AnnualSummaryDTO{ Year: ami.Year, AverageCount: ami.AverageCount, @@ -33,7 +23,6 @@ func NewAnnualSummaryDTO(ami models.AnnualSummary) *AnnualSummaryDTO { OtherRemunerations: ami.OtherRemunerations, Discounts: ami.Discounts, NumMonthsWithData: ami.NumMonthsWithData, - Timestamp: timestamp, } } @@ -46,6 +35,5 @@ func (ami *AnnualSummaryDTO) ConvertToModel() *models.AnnualSummary { OtherRemunerations: ami.OtherRemunerations, Discounts: ami.Discounts, NumMonthsWithData: ami.NumMonthsWithData, - Timestamp: timestamppb.New(ami.Timestamp), } } diff --git a/repo/database/postgres.go b/repo/database/postgres.go index 00f25be..6974c0c 100644 --- a/repo/database/postgres.go +++ b/repo/database/postgres.go @@ -222,7 +222,9 @@ func (p *PostgresDB) GetAgenciesByUF(uf string) ([]models.Agency, error) { func (p *PostgresDB) GetAgency(aid string) (*models.Agency, error) { var dtoOrgao dto.AgencyDTO aid = strings.ToLower(aid) - if err := p.db.Model(&dto.AgencyDTO{}).Where("id = ?", aid).First(&dtoOrgao).Error; err != nil { + m := p.db.Model(&dto.AgencyDTO{}).Select("orgaos.*, max(coletas.timestamp) as ultima_coleta") + m = m.Joins("INNER JOIN coletas ON atual = true AND coletas.id_orgao = orgaos.id AND orgaos.id = ?", aid) + if err := m.Group("orgaos.id").First(&dtoOrgao).Error; err != nil { return nil, fmt.Errorf("error getting agency '%s': %q", aid, err) } orgao, err := dtoOrgao.ConvertToModel() @@ -284,10 +286,9 @@ func (p *PostgresDB) GetAnnualSummary(agency string) ([]models.AnnualSummary, er SUM(CAST(sumario -> 'remuneracao_base' ->> 'total' AS DECIMAL)) AS remuneracao_base, SUM(CAST(sumario -> 'outras_remuneracoes' ->> 'total' AS DECIMAL)) AS outras_remuneracoes, SUM(CAST(sumario -> 'descontos' ->> 'total' AS DECIMAL)) AS descontos, - COUNT(*) FILTER (WHERE (procinfo::text = 'null' OR procinfo IS NULL)) AS meses_com_dados, - max(timestamp) as timestamp` + COUNT(*) AS meses_com_dados` m := p.db.Model(&dtoAgmi).Select(query) - m = m.Where("id_orgao = ? AND atual = TRUE ", agency) + m = m.Where("id_orgao = ? AND atual = TRUE AND (procinfo::text = 'null' OR procinfo IS NULL) ", agency) m = m.Group("ano, id_orgao").Order("ano ASC") if err := m.Scan(&dtoAmis).Error; err != nil { return nil, fmt.Errorf("error getting annual monthly info: %q", err)