diff --git a/core/api.go b/core/api.go index 326ae6e..c4b9202 100644 --- a/core/api.go +++ b/core/api.go @@ -1,13 +1,19 @@ package core import ( + "bytes" + "encoding/binary" + "encoding/hex" "encoding/json" "errors" "flag" "fmt" "io" "net/http" + "os" + "path" "sort" + "strconv" "strings" "sync/atomic" "time" @@ -21,6 +27,54 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) +type fileDocumentDesc struct { + Service string + Key string + Src string + Link string + Desc string + Extract bool + Timestamp int64 + Contents []byte `json:",omitempty"` +} + +func (fd *fileDocumentDesc) save() error { + // 새 파일 올라옴 + if len(fd.Contents) == 0 { + return nil + } + + var destFile string + if fd.Extract { + os.MkdirAll(fd.Link, os.ModePerm) + destFile = path.Join(fd.Link, fd.Src) + } else { + os.MkdirAll(path.Dir(fd.Link), os.ModePerm) + destFile = fd.Link + } + + f, err := os.Create(destFile) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(f, bytes.NewBuffer(fd.Contents)) + if err != nil { + return err + } + + if fd.Extract { + switch path.Ext(destFile) { + case ".zip": + err = common.Unzip(destFile) + case ".tar": + err = common.Untar(destFile) + } + } + return err +} + func (caller apiCaller) isGlobalAdmin() bool { if *noauth { return true @@ -60,13 +114,18 @@ func (caller apiCaller) writeAccessableServices(w http.ResponseWriter) { func (caller apiCaller) getAccessableServices() ([]*serviceDescription, []string) { allservices := caller.mg.services.all() - v, ok := caller.userinfo["email"] - if !ok { - return nil, nil - } - email := v.(string) - _, admin := caller.admins[email] + admin := caller.isGlobalAdmin() + var email string + if !*noauth { + v, ok := caller.userinfo["email"] + if !ok { + return nil, nil + } + + email := v.(string) + _, admin = caller.admins[email] + } var output []*serviceDescription var editable []string @@ -114,30 +173,79 @@ func (caller apiCaller) isValidUser(service any, category string) (valid bool, a return svcdesc.isValidAPIUser(category, email), false } +func (caller apiCaller) filesAPI(w http.ResponseWriter, r *http.Request) error { + serviceid := r.FormValue("service") + if len(serviceid) == 0 { + serviceid = "000000000000" + } + + var files []fileDocumentDesc + err := caller.mg.mongoClient.FindAllAs(CollectionFile, bson.M{ + "service": serviceid, + }, &files, options.Find().SetProjection(bson.M{ + "contents": 0, + })) + if err != nil { + return err + } + + enc := json.NewEncoder(w) + return enc.Encode(files) +} + +var seq = uint32(0) + func (caller apiCaller) uploadAPI(w http.ResponseWriter, r *http.Request) error { - // file, header, err := r.FormFile("file") - // if err != nil { - // logger.Error(err) - // w.WriteHeader(http.StatusBadRequest) - // return - // } - // defer file.Close() + serviceid := r.FormValue("service") + if len(serviceid) == 0 { + serviceid = "000000000000" + } - // contents, err := io.ReadAll(file) - // if err != nil { - // logger.Error(err) - // w.WriteHeader(http.StatusInternalServerError) - // return - // } + infile, header, err := r.FormFile("file") + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return err + } + defer infile.Close() - // ext := path.Ext(header.Filename) - // if ext == ".zip" { + desc := r.FormValue("desc") + contents, _ := io.ReadAll(infile) + extractstr := r.FormValue("extract") + extract, _ := strconv.ParseBool(extractstr) - // } + var b [5]byte + binary.BigEndian.PutUint32(b[0:4], uint32(time.Now().Unix())) + b[4] = byte(atomic.AddUint32(&seq, 1) % 255) + rf := hex.EncodeToString(b[:]) - // // deploys 폴더는 파일시스템 서비스이므로 다운로드 가능 - // filename := path.Join("deploys", name, version, name+ext) - return nil + var link string + if extract { + link = path.Join("static", serviceid, rf) + } else { + link = path.Join("static", serviceid, rf, header.Filename) + } + + newdoc := fileDocumentDesc{ + Contents: contents, + Src: header.Filename, + Timestamp: time.Now().UTC().Unix(), + Extract: extract, + Link: link, + Desc: desc, + Key: rf, + Service: serviceid, + } + _, _, err = caller.mg.mongoClient.UpsertOne(CollectionFile, bson.M{ + "service": serviceid, + "key": rf, + }, newdoc) + + if err == nil { + newdoc.Contents = nil + enc := json.NewEncoder(w) + enc.Encode(newdoc) + } + return err } func (caller apiCaller) whitelistAPI(w http.ResponseWriter, r *http.Request) error { @@ -215,25 +323,6 @@ func (caller apiCaller) whitelistAPI(w http.ResponseWriter, r *http.Request) err return nil } -// func (caller apiCaller) divisionAPI(w http.ResponseWriter, r *http.Request, svcid string, divid string) error { -// if r.Method == "PUT" { -// // svcid, divid에 statemeta 설정 -// file, header, err := r.FormFile("file") -// if err != nil { -// logger.Error(err) -// w.WriteHeader(http.StatusBadRequest) -// return -// } -// defer file.Close() - -// if header. -// stateFile, header, err := r.FormFile("file") -// if err != nil { -// return err -// } -// } -// } - func (caller apiCaller) serviceAPI(w http.ResponseWriter, r *http.Request) error { mg := caller.mg queryvals := r.URL.Query() @@ -277,6 +366,10 @@ func (caller apiCaller) serviceAPI(w http.ResponseWriter, r *http.Request) error } filter := bson.M{"_id": service.Id} + if len(service.ServiceCode) == 0 { + service.ServiceCode = hex.EncodeToString(service.Id[6:]) + } + success, _, err := mg.mongoClient.Update(CollectionService, filter, bson.M{ "$set": &service, }, options.Update().SetUpsert(true)) @@ -511,6 +604,8 @@ func (mg *Maingate) api(w http.ResponseWriter, r *http.Request) { err = caller.accountAPI(w, r) } else if strings.HasSuffix(r.URL.Path, "/upload") { err = caller.uploadAPI(w, r) + } else if strings.HasSuffix(r.URL.Path, "/files") { + err = caller.filesAPI(w, r) } if err != nil { diff --git a/core/maingate.go b/core/maingate.go index 453c359..a759307 100644 --- a/core/maingate.go +++ b/core/maingate.go @@ -39,6 +39,7 @@ var ( CollectionAuth = common.CollectionName("auth") CollectionWhitelist = common.CollectionName("whitelist") CollectionService = common.CollectionName("service") + CollectionFile = common.CollectionName("file") CollectionBlock = common.CollectionName("block") CollectionPlatformLoginToken = common.CollectionName("platform_login_token") //-- 각 플랫폼에 로그인 및 권한 받아오는 과정에 사용하는 Key CollectionUserToken = common.CollectionName("usertoken") @@ -434,6 +435,18 @@ func (mg *Maingate) prepare(context context.Context) (err error) { return err } + if err = mg.mongoClient.MakeIndices(CollectionFile, map[string]bson.D{ + "service": {{Key: "service", Value: 1}}, + }); err != nil { + return err + } + + if err = mg.mongoClient.MakeUniqueIndices(CollectionFile, map[string]bson.D{ + "sk": {{Key: "service", Value: 1}, {Key: "key", Value: 1}}, + }); err != nil { + return err + } + if err = mg.mongoClient.MakeExpireIndex(CollectionWhitelist, 10); err != nil { return err } @@ -482,6 +495,36 @@ func (mg *Maingate) prepare(context context.Context) (err error) { mg.auths = makeAuthCollection(mg.mongoClient, time.Duration(mg.SessionTTL*int64(time.Second))) + var preall []struct { + Link string `bson:"link"` + Id primitive.ObjectID `bson:"_id"` + } + if err = mg.mongoClient.FindAllAs(CollectionFile, nil, &preall, options.Find().SetProjection(bson.M{ + "link": 1, + })); err != nil { + return err + } + + for _, pre := range preall { + _, err := os.Stat(pre.Link) + if !os.IsNotExist(err) { + continue + } + logger.Println("saving files :", pre.Link) + + var fulldoc fileDocumentDesc + err = mg.mongoClient.FindOneAs(CollectionFile, bson.M{ + "_id": pre.Id, + }, &fulldoc) + if err != nil { + return err + } + err = fulldoc.save() + if err != nil { + return err + } + } + go watchAuthCollection(context, mg.auths, mg.mongoClient) go mg.watchWhitelistCollection(context) @@ -579,7 +622,7 @@ func (mg *Maingate) RegisterHandlers(ctx context.Context, serveMux *http.ServeMu serveMux.HandleFunc(common.MakeHttpHandlerPattern(prefix, "authorize_sdk", AuthPlatformFirebaseAuth), mg.platform_firebaseauth_authorize_sdk) go mg.watchServiceCollection(ctx, serveMux, prefix) - + go mg.watchFileCollection(ctx, serveMux, prefix) // fsx := http.FileServer(http.Dir("console")) // serveMux.Handle("/console/", http.StripPrefix("/console/", fsx)) // logger.Println("console file server open") diff --git a/core/service.go b/core/service.go index 1ebd74b..977d348 100644 --- a/core/service.go +++ b/core/service.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "path" "strings" "sync/atomic" "time" @@ -26,16 +27,25 @@ type blockinfo struct { Reason string } +type whitelistAuthType = string + +const ( + whitelistAuthType_Default = whitelistAuthType("") + whitelistAuthType_QA = whitelistAuthType("qa") +) + type whitelistmember struct { Service string Email string Platform string Desc string + Auth []whitelistAuthType Expired primitive.DateTime `bson:"_ts,omitempty" json:"_ts,omitempty"` } type whitelist struct { emailptr unsafe.Pointer + qaptr unsafe.Pointer working int32 } @@ -50,17 +60,33 @@ type usertokeninfo struct { } func (wl *whitelist) init(total []whitelistmember) { - next := make(map[string]*whitelistmember) + auths := make(map[string]map[string]*whitelistmember) for _, member := range total { - next[whitelistKey(member.Email)] = &member - } + all := auths[""] + if all == nil { + all = make(map[string]*whitelistmember) + auths[""] = all + } + all[whitelistKey(member.Email)] = &member - atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&next)) - atomic.StoreInt32(&wl.working, 1) + for _, auth := range member.Auth { + spec := auths[auth] + if spec == nil { + spec = make(map[string]*whitelistmember) + auths[auth] = spec + } + spec[whitelistKey(member.Email)] = &member + } + } + all := auths[whitelistAuthType_Default] + atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&all)) + + qa := auths[whitelistAuthType_QA] + atomic.StorePointer(&wl.qaptr, unsafe.Pointer(&qa)) } -func (wl *whitelist) add(m *whitelistmember) { - ptr := atomic.LoadPointer(&wl.emailptr) +func addToUnsafePointer(to *unsafe.Pointer, m *whitelistmember) { + ptr := atomic.LoadPointer(to) src := (*map[string]*whitelistmember)(ptr) next := map[string]*whitelistmember{} @@ -68,11 +94,11 @@ func (wl *whitelist) add(m *whitelistmember) { next[k] = v } next[whitelistKey(m.Email)] = m - atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&next)) + atomic.StorePointer(to, unsafe.Pointer(&next)) } -func (wl *whitelist) remove(email string) { - ptr := atomic.LoadPointer(&wl.emailptr) +func removeFromUnsafePointer(from *unsafe.Pointer, email string) { + ptr := atomic.LoadPointer(from) src := (*map[string]*whitelistmember)(ptr) next := make(map[string]*whitelistmember) @@ -80,7 +106,21 @@ func (wl *whitelist) remove(email string) { next[k] = v } delete(next, whitelistKey(email)) - atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&next)) + atomic.StorePointer(from, unsafe.Pointer(&next)) +} + +func (wl *whitelist) add(m *whitelistmember) { + addToUnsafePointer(&wl.emailptr, m) + for _, auth := range m.Auth { + if auth == whitelistAuthType_QA { + addToUnsafePointer(&wl.qaptr, m) + } + } +} + +func (wl *whitelist) remove(email string) { + removeFromUnsafePointer(&wl.emailptr, email) + removeFromUnsafePointer(&wl.qaptr, email) } func (wl *whitelist) isMember(email string, platform string) bool { @@ -97,30 +137,46 @@ func (wl *whitelist) isMember(email string, platform string) bool { return false } +func (wl *whitelist) hasAuth(email string, platform string, auth whitelistAuthType) bool { + if auth == whitelistAuthType_QA { + ptr := atomic.LoadPointer(&wl.qaptr) + src := *(*map[string]*whitelistmember)(ptr) + + if member, exists := src[whitelistKey(email)]; exists { + return member.Platform == platform + } + } + + return false +} + +type divisionStateName string + const ( - DivisionState_Closed = string("closed") - DivisionState_Maintenance = string("maintenance") - DivisionState_RestrictedOpen = string("restricted") - DivisionState_FullOpen = string("open") + DivisionState_Closed = divisionStateName("closed") + DivisionState_Maintenance = divisionStateName("maintenance") + DivisionState_RestrictedOpen = divisionStateName("restricted") + DivisionState_FullOpen = divisionStateName("open") ) type maintenance struct { - Link string - StartTime primitive.Timestamp + Notice string `bson:"notice"` + StartTimeUTC int64 `bson:"start_unixtime_utc" json:"start_unixtime_utc"` + link string } type division struct { - Url string `bson:"url"` - Priority int `bson:"priority"` - State string `bson:"state"` - Maintenance maintenance `bson:"maintenance"` + Url string `bson:"url"` + Priority int `bson:"priority"` + State divisionStateName `bson:"state"` + Maintenance *maintenance `bson:"maintenance"` } type serviceDescription struct { // sync.Mutex Id primitive.ObjectID `bson:"_id"` ServiceName string `bson:"service"` - Divisions map[string]division `bson:"divisions"` + Divisions map[string]*division `bson:"divisions"` ServiceCode string `bson:"code"` UseWhitelist bool `bson:"use_whitelist"` Closed bool `bson:"closed"` @@ -175,12 +231,46 @@ func (sh *serviceDescription) readProfile(authtype string, id string, binfo stri } func (sh *serviceDescription) prepare(mg *Maingate) error { - div := sh.Divisions + divs := sh.Divisions if len(sh.ServiceCode) == 0 { sh.ServiceCode = hex.EncodeToString(sh.Id[6:]) } - divmarshaled, _ := json.Marshal(div) + var closed []string + for dn, div := range divs { + if div.State == DivisionState_Closed { + closed = append(closed, dn) + continue + } + + if len(div.State) == 0 { + div.State = DivisionState_FullOpen + } + + if div.State != DivisionState_FullOpen { + if div.Maintenance == nil { + div.Maintenance = &maintenance{} + } + + if len(div.Maintenance.link) == 0 { + if len(div.Maintenance.Notice) == 0 { + div.Maintenance.link = "https://www.action2quare.com" + } else if strings.HasPrefix(div.Maintenance.Notice, "http") { + div.Maintenance.link = div.Maintenance.Notice + } else { + div.Maintenance.link = path.Join("static", sh.ServiceCode, div.Maintenance.Notice) + } + } + } else { + div.Maintenance = nil + } + } + + for _, dn := range closed { + delete(divs, dn) + } + + divmarshaled, _ := json.Marshal(divs) devstr := string(divmarshaled) sh.divisionsSerialized = unsafe.Pointer(&devstr) @@ -200,15 +290,17 @@ func (sh *serviceDescription) prepare(mg *Maingate) error { sh.closed = 0 } - if sh.UseWhitelist { - var whites []whitelistmember - if err := mg.mongoClient.FindAllAs(CollectionWhitelist, bson.M{ - "$or": []bson.M{{"service": sh.ServiceName}, {"service": sh.ServiceCode}}, - }, &whites, options.Find().SetReturnKey(false)); err != nil { - return err - } + var whites []whitelistmember + if err := mg.mongoClient.FindAllAs(CollectionWhitelist, bson.M{ + "$or": []bson.M{{"service": sh.ServiceName}, {"service": sh.ServiceCode}}, + }, &whites, options.Find().SetReturnKey(false)); err != nil { + return err + } - sh.wl.init(whites) + sh.wl.init(whites) + + if sh.UseWhitelist { + sh.wl.working = 1 } else { sh.wl.working = 0 } @@ -517,9 +609,9 @@ func (sh *serviceDescription) authorize(w http.ResponseWriter, r *http.Request) ServiceCode: sh.ServiceCode, Platform: authtype, Uid: uid, - //Token: accesstoken, - Sk: newsession, - Expired: expired, + Email: email, + Sk: newsession, + Expired: expired, //RefreshToken: queryvals.Get("rt"), } @@ -604,8 +696,6 @@ func (sh *serviceDescription) ServeHTTP(w http.ResponseWriter, r *http.Request) } else { // TODO : 세션키와 authtoken을 헤더로 받아서 accid 조회 queryvals := r.URL.Query() - //token := queryvals.Get("token") - token := "" // 더이상 쓰지 않는다. sk := queryvals.Get("sk") //if len(token) == 0 || len(sk) == 0 { @@ -617,14 +707,53 @@ func (sh *serviceDescription) ServeHTTP(w http.ResponseWriter, r *http.Request) // TODO : 각 서버에 있는 자산? 캐릭터 정보를 보여줘야 하나. 뭘 보여줄지는 프로젝트에 문의 // 일단 서버 종류만 내려보내자 // 세션키가 있는지 확인 - if _, ok := sh.auths.IsValid(sk, token); !ok { - logger.Println("sessionkey is not valid :", sk, token) + if _, ok := sh.auths.IsValid(sk, ""); !ok { + logger.Println("sessionkey is not valid :", sk) w.WriteHeader(http.StatusBadRequest) return } - divstrptr := atomic.LoadPointer(&sh.divisionsSerialized) - divstr := *(*string)(divstrptr) - w.Write([]byte(divstr)) + if divname := queryvals.Get("div"); len(divname) > 0 { + divname = strings.Trim(divname, `"`) + // 점검중인지 아닌지 확인 + // 점검중이어도 입장이 가능한 인원이 있다. + div := sh.Divisions[divname] + if div != nil { + switch div.State { + case DivisionState_FullOpen: + w.Write([]byte(fmt.Sprintf(`{"service":"%s"}`, div.Url))) + + case DivisionState_RestrictedOpen: + // 점검중인데 일부 권한을 갖고 있는 유저만 들어갈 수 있는 상태 + cell := sh.auths.QuerySession(sk, "") + if cell == nil { + logger.Println("sessionkey is not valid :", sk) + w.WriteHeader(http.StatusBadRequest) + return + } + if sh.wl.hasAuth(cell.ToAuthinfo().Email, cell.ToAuthinfo().Platform, whitelistAuthType_QA) { + // qa 권한이면 입장 가능 + w.Write([]byte(fmt.Sprintf(`{"service":"%s"}`, div.Url))) + } else if div.Maintenance != nil { + // 권한이 없으므로 공지 + w.Write([]byte(fmt.Sprintf(`{"notice":"%s"}`, div.Maintenance.link))) + } else { + logger.Println("div.Maintenance is nil :", divname) + } + + case DivisionState_Maintenance: + // 점검중. 아무도 못들어감 + if div.Maintenance != nil { + w.Write([]byte(fmt.Sprintf(`{"notice":"%s"}`, div.Maintenance.link))) + } else { + logger.Println("div.Maintenance is nil :", divname) + } + } + } + } else { + divstrptr := atomic.LoadPointer(&sh.divisionsSerialized) + divstr := *(*string)(divstrptr) + w.Write([]byte(divstr)) + } } } diff --git a/core/watch.go b/core/watch.go index 4e8516f..44ee9f2 100644 --- a/core/watch.go +++ b/core/watch.go @@ -31,6 +31,10 @@ type servicePipelineDocument struct { Service *serviceDescription `bson:"fullDocument"` } +type filePipelineDocument struct { + File *fileDocumentDesc `bson:"fullDocument"` +} + type whilelistPipelineDocument struct { OperationType string `bson:"operationType"` DocumentKey struct { @@ -123,6 +127,70 @@ func (mg *Maingate) watchWhitelistCollection(parentctx context.Context) { } } +func (mg *Maingate) watchFileCollection(parentctx context.Context, serveMux *http.ServeMux, prefix string) { + defer func() { + s := recover() + if s != nil { + logger.Error(s) + } + }() + + matchStage := bson.D{ + { + Key: "$match", Value: bson.D{ + {Key: "operationType", Value: bson.D{ + {Key: "$in", Value: bson.A{ + "insert", + }}, + }}, + }, + }} + projectStage := bson.D{ + { + Key: "$project", Value: bson.D{ + {Key: "fullDocument", Value: 1}, + }, + }, + } + var stream *mongo.ChangeStream + var err error + var ctx context.Context + + for { + if stream == nil { + stream, err = mg.mongoClient.Watch(CollectionFile, mongo.Pipeline{matchStage, projectStage}, options.ChangeStream().SetFullDocument(options.UpdateLookup)) + if err != nil { + logger.Error("watchFileCollection watch failed :", err) + time.Sleep(time.Second) + continue + } + ctx = context.TODO() + } + + changed := stream.TryNext(ctx) + if ctx.Err() != nil { + logger.Error("watchFileCollection stream.TryNext failed. process should be restarted! :", ctx.Err().Error()) + break + } + + if !changed { + if stream.Err() != nil || stream.ID() == 0 { + logger.Error("watchServiceCollection stream error :", stream.Err()) + stream.Close(ctx) + stream = nil + } else { + time.Sleep(time.Second) + } + continue + } + + var data filePipelineDocument + if err := stream.Decode(&data); err == nil { + data.File.save() + } + } +} + func (mg *Maingate) watchServiceCollection(parentctx context.Context, serveMux *http.ServeMux, prefix string) { defer func() { s := recover() diff --git a/go.mod b/go.mod index a5d84e4..50ef594 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible go.mongodb.org/mongo-driver v1.11.6 google.golang.org/api v0.123.0 - repositories.action2quare.com/ayo/gocommon v0.0.0-20230524061015-e95efa06a6d4 + repositories.action2quare.com/ayo/gocommon v0.0.0-20230528100715-93bd4f6c0bab ) require ( diff --git a/go.sum b/go.sum index f8575b0..b64c1ce 100644 --- a/go.sum +++ b/go.sum @@ -260,3 +260,7 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= repositories.action2quare.com/ayo/gocommon v0.0.0-20230524061015-e95efa06a6d4 h1:DFrkLvbPWqwVDU4X0QGJs2lhPduJYJU+JM/r1L2RMwo= repositories.action2quare.com/ayo/gocommon v0.0.0-20230524061015-e95efa06a6d4/go.mod h1:pw573a06qV7dP1lSyavbWmzyYAsmwtK6mdbFENbh3cs= +repositories.action2quare.com/ayo/gocommon v0.0.0-20230524093812-0acae49a22e7 h1:4U70jZtyMQpcF1T8z/HU8LOR2/MXoF2eJvun6lbnyuo= +repositories.action2quare.com/ayo/gocommon v0.0.0-20230524093812-0acae49a22e7/go.mod h1:5RmALPCFGFmqXa+AAPLsQaSlBVBafwX1H2CnIhsCM50= +repositories.action2quare.com/ayo/gocommon v0.0.0-20230528100715-93bd4f6c0bab h1:EMlxwDayv3rn8ttJcJuDLYoHA5odVn85+LjdAuw+2dw= +repositories.action2quare.com/ayo/gocommon v0.0.0-20230528100715-93bd4f6c0bab/go.mod h1:ng62uGMGXyQSeuxePG5gJAMtip4Rnspu5Tu7hgvaXns=