diff --git a/cmd/errors.go b/cmd/errors.go index 297c07b..1b0385b 100644 --- a/cmd/errors.go +++ b/cmd/errors.go @@ -25,7 +25,6 @@ func checkError(err error) { errors.Is(err, promptui.ErrInterrupt) { os.Exit(1) } else if errors.As(err, &eg) { - logger.Error(err, eg.ResponseString) exitWithMsg(err.Error()) } else if errors.Is(err, geektime.ErrWrongPassword) || errors.Is(err, geektime.ErrTooManyLoginAttemptTimes) { diff --git a/internal/config/config.go b/internal/config/config.go index b5660ae..78f01d9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,7 @@ func ReadCookieFromConfigFile(phone string) ([]*http.Cookie, error) { if len(files) == 0 { return nil, nil } + now := time.Now() for _, fi := range files { if fi.IsDir() { continue @@ -43,7 +44,6 @@ func ReadCookieFromConfigFile(phone string) ([]*http.Cookie, error) { if strings.HasPrefix(fi.Name(), phone) { fullName := filepath.Join(userConfigDir, GeektimeDownloaderFolder, fi.Name()) var cookies []*http.Cookie - oneyear := time.Now().Add(180 * 24 * time.Hour) data, err := ioutil.ReadFile(fullName) if err != nil { @@ -51,16 +51,20 @@ func ReadCookieFromConfigFile(phone string) ([]*http.Cookie, error) { } for _, line := range strings.Split(string(data), "\n") { - s := strings.SplitN(line, " ", 2) - if len(s) != 2 { + s := strings.SplitN(line, " ", 3) + if len(s) != 3 { continue } + t, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", s[2]) + if err != nil || t.Before(now) { + break + } cookies = append(cookies, &http.Cookie{ Name: s[0], Value: s[1], Domain: geektime.GeekBangCookieDomain, HttpOnly: true, - Expires: oneyear, + Expires: t, }) } return cookies, nil @@ -75,16 +79,8 @@ func WriteCookieToConfigFile(phone string, cookies []*http.Cookie) error { if err := os.MkdirAll(dir, os.ModePerm); err != nil { return err } - files, err := ioutil.ReadDir(dir) - if err != nil { - return err - } - for _, fi := range files { - // config file already exists - if strings.HasPrefix(fi.Name(), phone) { - return nil - } - } + removeConfig(dir, phone) + file, err := ioutil.TempFile(dir, phone) if err != nil { return err @@ -92,7 +88,7 @@ func WriteCookieToConfigFile(phone string, cookies []*http.Cookie) error { defer file.Close() var sb strings.Builder for _, v := range cookies { - sb = writeOnelineConfig(sb, v.Name, v.Value) + sb = writeOnelineConfig(sb, v) } if _, err := file.Write([]byte(sb.String())); err != nil { return err @@ -103,6 +99,10 @@ func WriteCookieToConfigFile(phone string, cookies []*http.Cookie) error { // RemoveConfig remove specified users' config func RemoveConfig(phone string) error { dir := filepath.Join(userConfigDir, GeektimeDownloaderFolder) + return removeConfig(dir, phone) +} + +func removeConfig(dir, phone string) error { files, err := ioutil.ReadDir(dir) if err != nil { return err @@ -121,13 +121,15 @@ func RemoveConfig(phone string) error { } } } - return nil + return nil; } -func writeOnelineConfig(sb strings.Builder, key string, value string) strings.Builder { - sb.WriteString(key) +func writeOnelineConfig(sb strings.Builder, cookie *http.Cookie) strings.Builder { + sb.WriteString(cookie.Name) + sb.WriteString(" ") + sb.WriteString(cookie.Value) sb.WriteString(" ") - sb.WriteString(value) + sb.WriteString(cookie.Expires.String()) sb.WriteString("\n") return sb } diff --git a/internal/geektime/geektime.go b/internal/geektime/geektime.go index 1b46fba..4f2e55f 100644 --- a/internal/geektime/geektime.go +++ b/internal/geektime/geektime.go @@ -167,12 +167,21 @@ func (c *Client) Login(phone, password string) ([]*http.Cookie, error) { }, &res, ) + + logger.Infof("Login request start") resp, err := r.Execute(r.Method, r.URL) if err != nil { return nil, err } + if resp.RawResponse.StatusCode != 200 || res.Code != 0 { + logger.Warnf("Login request end, status code: %d, response body: %s", + resp.RawResponse.StatusCode, + resp.String(), + ) + } + if res.Code == 0 { var cookies []*http.Cookie for _, c := range resp.Cookies() { @@ -206,13 +215,20 @@ func (c *Client) Auth(cs []*http.Cookie) error { res, ) r.SetHeader(Origin, DefaultBaseURL) + + logger.Infof("Auth request start") resp, err := r.Execute(r.Method, r.URL) if err != nil { return err } - if resp.RawResponse.StatusCode == 452 || res.Code != 0 { + if resp.RawResponse.StatusCode != 200 || res.Code != 0 { + logger.Warnf("Auth request end, status code: %d, response body: %s", + resp.RawResponse.StatusCode, + resp.String(), + ) + // result Code -1 // {\"error\":{\"msg\":\"未登录\",\"code\":-2000} return ErrAuthFailed @@ -394,10 +410,10 @@ func (c *Client) MyClassProduct(classID int) (Product, error) { } p = Product{ - Access: true, - ID: classID, - Title: res.Data.Title, - Type: "", + Access: true, + ID: classID, + Title: res.Data.Title, + Type: "", IsVideo: true, } var articles []Article @@ -433,16 +449,24 @@ func (c *Client) newRequest(method, url string, params map[string]string, body i } func do(r *resty.Request) (*resty.Response, error) { + logger.Infof("Http request start, method: %s, url: %s", + r.Method, + r.URL, + ) resp, err := r.Execute(r.Method, r.URL) if err != nil { return nil, err } - if resp.RawResponse.StatusCode == 451 { - return nil, ErrGeekTimeRateLimit - } else if resp.RawResponse.StatusCode == 452 { - return nil, ErrAuthFailed + statusCode := resp.RawResponse.StatusCode + if statusCode != 200 { + logNotOkResponse(resp) + if statusCode == 451 { + return nil, ErrGeekTimeRateLimit + } else if statusCode == 452 { + return nil, ErrAuthFailed + } } rv := reflect.ValueOf(r.Result) @@ -453,5 +477,20 @@ func do(r *resty.Request) (*resty.Response, error) { return resp, nil } + logNotOkResponse(resp) + //未登录或者已失效 + if code == -3050 || code == -2000 { + return nil, ErrAuthFailed + } + return nil, ErrGeekTimeAPIBadCode{r.URL, resp.String()} } + +func logNotOkResponse(resp *resty.Response) { + logger.Warnf("Http request end, method: %s, url: %s, status code: %d, response body: %s", + resp.RawResponse.Request.Method, + resp.RawResponse.Request.URL, + resp.RawResponse.StatusCode, + resp.String(), + ) +} diff --git a/internal/pdf/pdf.go b/internal/pdf/pdf.go index f69d096..872c4c0 100644 --- a/internal/pdf/pdf.go +++ b/internal/pdf/pdf.go @@ -93,6 +93,14 @@ func hideRedundantElements(downloadComments bool) chromedp.ActionFunc { return chromedp.ActionFunc(func(ctx context.Context) error { s := ` + var headMain = document.getElementsByClassName('main')[0]; + if(headMain){ + headMain.style.display="none"; + } + var bottomWrapper = document.getElementsByClassName('sub-bottom-wrapper')[0]; + if(bottomWrapper){ + bottomWrapper.style.display="none"; + } var openAppdiv = document.getElementsByClassName('openApp')[0]; if(openAppdiv){ openAppdiv.parentNode.parentNode.parentNode.style.display="none"; @@ -177,4 +185,4 @@ func printToPDF(res *[]byte) chromedp.ActionFunc { *res = data return nil }) -} \ No newline at end of file +} diff --git a/internal/pkg/downloader/downloader.go b/internal/pkg/downloader/downloader.go index 0cfeddd..f9e00bb 100644 --- a/internal/pkg/downloader/downloader.go +++ b/internal/pkg/downloader/downloader.go @@ -135,7 +135,7 @@ func retry(attempts int, sleep time.Duration, f func() error) (err error) { time.Sleep(sleep) sleep *= 2 - logger.Info("retry hanppen, times: " + strconv.Itoa(i)) + logger.Infof("retry hanppen, times: %s", strconv.Itoa(i)) } err = f() if err == nil || errors.Is(err, context.Canceled) { diff --git a/internal/pkg/logger/logger.go b/internal/pkg/logger/logger.go index 4940473..18a38e5 100644 --- a/internal/pkg/logger/logger.go +++ b/internal/pkg/logger/logger.go @@ -1,8 +1,10 @@ package logger import ( + "fmt" "os" "path/filepath" + "runtime" "github.com/sirupsen/logrus" ) @@ -16,13 +18,36 @@ var ( logger = logrus.New() ) +type customFormatter struct { +} + +// Format custom logrus log format +func (f *customFormatter) Format(entry *logrus.Entry) ([]byte, error) { + // Get the file and line number where the log was called + _, filename, line, _ := runtime.Caller(7) + + // Get the script name from the full file path + fullPathName := filepath.Base(filename) + + // Format the log message + message := fmt.Sprintf("[%s] [%s] [%s:%d] %s\n", + entry.Time.Format("2006-01-02 15:04:05"), // Date-time + entry.Level.String(), // Log level + fullPathName, // Full path name + line, // Line number + entry.Message, // Log message + ) + + return []byte(message), nil +} + func init(){ userConfigDir, _ := os.UserConfigDir() logFilePath := filepath.Join(userConfigDir, GeektimeLogFolder, GeektimeLogFolder + ".log") - logger.SetFormatter(&logrus.TextFormatter{}) - logger.SetReportCaller(false) - logger.SetLevel(logrus.WarnLevel) + logger.SetReportCaller(true) + logger.SetLevel(logrus.InfoLevel) + logger.SetFormatter(&customFormatter{}) logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err == nil { logger.Out = logFile @@ -32,24 +57,14 @@ func init(){ logger.SetOutput(logFile) } -// Trace wrapper logrus log.Trace -func Trace(args ...interface{}) { - logger.Log(logrus.TraceLevel, args...) -} - -// Debug wrapper logrus log.Debug -func Debug(args ...interface{}) { - logger.Log(logrus.DebugLevel, args...) -} - -// Info wrapper logrus log.Info -func Info(args ...interface{}) { - logger.Log(logrus.InfoLevel, args...) +// Infof wrapper logrus log.Infof +func Infof(format string, args ...interface{}) { + logger.Logf(logrus.InfoLevel, format, args...) } -// Warn wrapper logrus log.Warn -func Warn(args ...interface{}) { - logger.Log(logrus.WarnLevel, args...) +// Warnf wrapper logrus log.Warnf +func Warnf(format string, args ...interface{}) { + logger.Logf(logrus.WarnLevel, format, args...) } // Error wrapper logrus log.Error diff --git a/internal/pkg/m3u8/m3u8.go b/internal/pkg/m3u8/m3u8.go index c723206..bd1b2a1 100644 --- a/internal/pkg/m3u8/m3u8.go +++ b/internal/pkg/m3u8/m3u8.go @@ -13,11 +13,11 @@ var ( linePattern = regexp.MustCompile(`([a-zA-Z-]+)=("[^"]+"|[^",]+)`) ) -// Parse do m3u8 url GET request, and extract ts file names and decrypt key from that -func Parse(client *geektime.Client, m3u8url string) (tsFileNames []string, keyURI string, err error) { +// Parse do m3u8 url GET request, and extract ts file names and check if it's encrypt video +func Parse(client *geektime.Client, m3u8url string) (tsFileNames []string, isVodEncryptVideo bool, err error) { m3u8Resp, err := client.HTTPClient.R().SetDoNotParseResponse(true).Get(m3u8url) if err != nil { - return nil, "", err + return nil, false, err } defer m3u8Resp.RawBody().Close() s := bufio.NewScanner(m3u8Resp.RawBody()) @@ -33,7 +33,7 @@ func Parse(client *geektime.Client, m3u8url string) (tsFileNames []string, keyUR if strings.HasPrefix(line, "#EXT-X-KEY") && !gotKeyURI { // ONLY Method and URI, IV not present params := parseLineParameters(line) - keyURI, gotKeyURI = params["URI"], true + isVodEncryptVideo, gotKeyURI = params["MEATHOD"] == "AES-128", true } if !strings.HasPrefix(line, "#") && strings.HasSuffix(line, ".ts") { tsFileNames = append(tsFileNames, line) diff --git a/internal/video/video.go b/internal/video/video.go index b2d86ca..45f3b19 100644 --- a/internal/video/video.go +++ b/internal/video/video.go @@ -22,7 +22,7 @@ import ( ) const ( - syncByte = uint8(71) //0x47 + // syncByte = uint8(71) //0x47 // TSExtension ... TSExtension = ".ts" ) @@ -30,12 +30,12 @@ const ( // EncryptType enum type EncryptType int -const ( - // AliyunVodEncrypt ... - AliyunVodEncrypt EncryptType = iota - // HLSStandardEncrypt ... - HLSStandardEncrypt -) +// const ( +// // AliyunVodEncrypt ... +// AliyunVodEncrypt EncryptType = iota +// // HLSStandardEncrypt ... +// HLSStandardEncrypt +// ) // GetPlayInfoResponse is the response struct for api GetPlayInfo type GetPlayInfoResponse struct { @@ -120,13 +120,17 @@ func downloadAliyunVodEncryptVideo(ctx context.Context, return err } tsURLPrefix := extractTSURLPrefix(playInfo.PlayURL) - // just ignore keyURI in m3u8, aliyun private vod use another decrypt method - tsFileNames, _, err := m3u8.Parse(client, playInfo.PlayURL) + + tsFileNames, isVodEncryptVideo, err := m3u8.Parse(client, playInfo.PlayURL) if err != nil { return err } - decryptKey := crypto.GetAESDecryptKey(clientRand, playInfo.Rand, playInfo.Plaintext) - return download(ctx, tsURLPrefix, videoTitle, projectDir, tsFileNames, []byte(decryptKey), playInfo.Size, AliyunVodEncrypt, concurrency) + + decryptKey := "" + if isVodEncryptVideo { + decryptKey = crypto.GetAESDecryptKey(clientRand, playInfo.Rand, playInfo.Plaintext) + } + return download(ctx, tsURLPrefix, videoTitle, projectDir, tsFileNames, []byte(decryptKey), playInfo.Size, isVodEncryptVideo, concurrency) } // DownloadMP4 ... @@ -161,7 +165,7 @@ func download(ctx context.Context, tsFileNames []string, decryptKey []byte, size int64, - videoEncryptType EncryptType, + isVodEncryptVideo bool, concurrency int) (err error) { // Make temp ts folder and download temp ts files @@ -197,12 +201,12 @@ func download(ctx context.Context, bar.Finish() // Read temp ts files, decrypt and merge into the one final video file - err = mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir, decryptKey, videoEncryptType) + err = mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir, decryptKey, isVodEncryptVideo) return } -func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte, videoEncryptType EncryptType) error { +func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte, isVodEncryptVideo bool) error { tempTSFiles, err := ioutil.ReadDir(tempVideoDir) if err != nil { return err @@ -220,21 +224,23 @@ func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte, if err != nil { return err } - switch videoEncryptType { - case HLSStandardEncrypt: - aes128 := crypto.AESDecryptCBC(f, key, make([]byte, 16)) - // https://en.wikipedia.org/wiki/MPEG_transport_stream - for j := 0; j < len(aes128); j++ { - if aes128[j] == syncByte { - aes128 = aes128[j:] - break - } - } - f = aes128 - case AliyunVodEncrypt: + + if isVodEncryptVideo { tsParser := m3u8.NewTSParser(f, string(key)) f = tsParser.Decrypt() } + + // case HLSStandardEncrypt: + // aes128 := crypto.AESDecryptCBC(f, key, make([]byte, 16)) + // // https://en.wikipedia.org/wiki/MPEG_transport_stream + // for j := 0; j < len(aes128); j++ { + // if aes128[j] == syncByte { + // aes128 = aes128[j:] + // break + // } + // } + // f = aes128 + if _, err := finalVideoFile.Write(f); err != nil { return err }