ImagePool

imagePoolCode/ImagePoolProject/DatabaseQuery.go

@@ -0,0 +1,18 @@
+package main
+import (
+	"fmt"
+	"github.com/astaxie/beego/orm"
+)
+
+func init()  {
+}
+
+func GroupUrlGet(Group string){
+	o:=orm.NewOrm()
+	var obj []interface{}
+	_,err:=o.Raw("select imageID from imageLabelTable where labelName = ? and confidence=1 limit 10;",Group).QueryRows(&obj)
+	if err != nil {
+		fmt.Println(err)
+	}
+	fmt.Printf("%#v",obj)
+}

imagePoolCode/ImagePoolProject/Downloader.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"github.com/astaxie/beego/httplib"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+const (
+	ChanSize int = 5
+)
+
+//图片下载函数，保存图片到TempImage文件夹下
+func ImageDownload(imageID string ,url string,label string)(status bool,err error){
+	resp,err := http.Get(url)
+	if err != nil {
+		fmt.Println(err)
+		return false,errors.New("can't connected")
+	}
+	if resp.StatusCode==200{
+		img,err:=os.OpenFile("./TempImage/"+label+"/"+imageID+".jpg",os.O_WRONLY|os.O_CREATE,0666)
+		defer img.Close()
+		if err != nil {
+			println(err)
+		}
+		image:=make([]byte,10240)
+		for {
+			n,err := resp.Body.Read(image)
+			if err == io.EOF {
+				break
+			}
+			_,err = img.Write(image[:n])
+			if err != nil {
+				fmt.Println("img write:",err)
+			}
+		}
+		return true ,nil
+	} else{
+		err:=errors.New("StatusCode Error")
+		return false,err
+	}
+}
+func ImageDownload2(count chan int,url string,label string)(status bool,err error){
+	resp,err := http.Get(url)
+	if err != nil {
+		fmt.Println(err)
+		return false,errors.New("can't connected")
+	}
+	if resp.StatusCode==200{
+		a:=<-count
+		println(a,url)
+		img,err:=os.OpenFile("./TempImage/"+label+"/"+strconv.Itoa(a)+".jpg",os.O_WRONLY|os.O_CREATE,0777)
+		defer img.Close()
+		if err != nil {
+			println(err)
+		}
+		image:=make([]byte,256)
+		for {
+			n,err := resp.Body.Read(image)
+			if err == io.EOF {
+				break
+			}
+			_,err = img.Write(image[:n])
+			if err != nil {
+				fmt.Println("img write:",err)
+			}
+		}
+		return true ,nil
+	} else{
+		err:=errors.New("StatusCode Error")
+		return false,err
+	}
+}
+func urlDownload(url string)[]string{
+
+	resp:=httplib.Get(url)
+	text,err:=resp.String()
+	if err != nil {
+		fmt.Println(err)
+	}
+	urls:=strings.Split(text,"\n")
+	return urls
+}
+func ImageNetDownload(urls []string,label string)(int){
+	println(len(urls))
+	w:=sync.WaitGroup{}
+	lock:=sync.Mutex{}
+	sum :=9
+	count:=make(chan int,ChanSize)
+	for i:=0;i<ChanSize;i++{
+		count<-i
+	}
+	for i:=0;i<len(urls);i++{
+		w.Add(1)
+		go func(i int,url string,imgnum *int) {
+			url=strings.ReplaceAll(url,"\r","")
+			flag,err:=ImageDownload2(count,url,label)
+			if err != nil {
+				fmt.Printf("%v\n",err)
+			}
+			if flag==true{
+				lock.Lock()
+				*imgnum++
+				count<-*imgnum
+				lock.Unlock()
+			}
+			w.Done()
+		}(i,urls[i],&sum)
+	}
+	w.Wait()
+	return sum
+}

imagePoolCode/ImagePoolProject/conf/config.json

@@ -0,0 +1,3 @@
+{
+  "size":
+}

imagePoolCode/ImagePoolProject/filecontorl.go

@@ -0,0 +1,42 @@
+package main
+
+
+import (
+	"github.com/astaxie/beego/httplib"
+	"errors"
+)
+var Url ="http://127.0.0.1:8080/group1"
+func init(){}
+
+func FileUploadToDFS(imageID string,group string) (md5 string,err error){
+	obj:=make(map[string]string,10)
+	req:=httplib.Post(Url+"/upload")
+	req.PostFile("file","./TempImage/"+imageID+".jpg")//注意不是全路径
+	req.Param("output","json")
+	req.Param("scene",group)
+	req.Param("path","")
+	req.ToJSON(&obj)
+	return obj["md5"] ,nil
+}
+func FileDeleteFromDFS(md5 string,group string)(bool,error){
+	var obj interface{}
+	req:=httplib.Get(Url+"/delete?md5="+md5)
+	req.ToJSON(&obj)
+	stat:=obj.(map[string]interface{})["status"].(string)
+	if stat=="ok"{
+		return true ,nil
+	}else if stat=="false"{
+		return false ,	errors.New("File Already Delete!")
+	}
+	return false,nil
+}
+func FileStat()(num int,size int){
+	var obj interface{}
+	mp :=make(map[string]int,10)
+	mp["a"]=1
+	req:=httplib.Get("http://127.0.0.1:8080/group1/stat")
+	req.ToJSON(&obj)
+	fileNum:=int(obj.(map[string]interface{})["data"].([]interface{})[0].(map[string]interface{})["fileCount"].(float64))
+	fileSize:=int(obj.(map[string]interface{})["data"].([]interface{})[0].(map[string]interface{})["totalSize"].(float64))
+	return fileNum,fileSize
+}

imagePoolCode/ImagePoolProject/go.mod

@@ -0,0 +1,9 @@
+module ImagePoolProject
+
+go 1.13
+
+require (
+	github.com/astaxie/beego v1.12.0
+	github.com/go-redis/redis v6.15.6+incompatible
+	github.com/go-sql-driver/mysql v1.4.1
+)

imagePoolCode/ImagePoolProject/go.sum

@@ -0,0 +1,39 @@
+github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM=
+github.com/astaxie/beego v1.12.0 h1:MRhVoeeye5N+Flul5PoVfD9CslfdoH+xqC/xvSQ5u2Y=
+github.com/astaxie/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o=
+github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
+github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU=
+github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
+github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
+github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U=
+github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
+github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
+github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY=
+github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
+github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg=
+github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
+github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg=
+github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA=
+github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE=
+github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
+golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

imagePoolCode/ImagePoolProject/main.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	_ "github.com/astaxie/beego/orm"
+	_ "github.com/go-redis/redis"
+	_ "github.com/go-sql-driver/mysql"
+	"os"
+	"strconv"
+	"strings"
+)
+
+type ModelInit struct {
+	Id   int
+}
+
+//func init(){
+//	orm.RegisterDataBase("default", "mysql", "root:116118@tcp(www.whatdoyoudo.club:3306)/dataset?charset=utf8", 30)
+//	// create table
+//	orm.RegisterModel(new(ModelInit))
+//	orm.RunSyncdb("default", false, true)
+//}
+
+func ImageNetDownloader(){
+	arg := os.Args
+	label := arg[1]
+	pwd,err:= os.Getwd()
+	if err != nil {
+
+	}
+	println(pwd)
+	err=os.MkdirAll(pwd+"TempImage",0777)
+	if err != nil {
+		println(err)
+	}
+	err=os.MkdirAll(pwd+"/TempImage/"+label,0777)
+	if err != nil {
+		println(err)
+	}
+	urls :=urlDownload("http://image-net.org/api/text/imagenet.synset.geturls?wnid="+label)
+	n:=ImageNetDownload(urls,label)
+	println("下载",n-10,"条"+label+"图像")
+}
+
+func ImageNetDownloader2(){
+	arg := os.Args
+	label := arg[1]
+	pwd,err:= os.Getwd()
+	if err != nil {
+
+	}
+	println(pwd)
+	err=os.MkdirAll(pwd+"TempImage",0777)
+	if err != nil {
+		println(err)
+	}
+	err=os.MkdirAll(pwd+"/TempImage/"+label,0777)
+	if err != nil {
+		println(err)
+	}
+	urls :=urlDownload("http://image-net.org/api/text/imagenet.synset.geturls?wnid="+label)
+	for i,k:=range(urls){
+		println(i,k)
+		k=strings.ReplaceAll(k,"\r","")
+		ImageDownload(strconv.Itoa(i),k,label)
+	}
+}
+
+func main(){
+	ImageNetDownloader()
+}