Channel Stuck issue

[Wei Ding]

Hi guys,
I'm working on writing an exercise of concurrently downloading a file (from HTTP file server) using go routine. However I got stuck when debugging channel code.
Could you please help me take a look and see where I made wrong?  Thank you in advance.

Code:

package main

import (
    "fmt"
    "os"
    "net/http"
    "io/ioutil"
    "strconv"
    "bytes"

)


func downRange(ran, url string, data chan *[]byte) {
   
    fmt.Println("Downloading range: " + ran)
    req, _ := http.NewRequest("GET", url, nil)

    req.Header.Add("Range", "bytes= " + ran)

    // fmt.Println(req.Header)

    client := &http.Client{}
   
    resp, err1:= client.Do(req)

    if err1 != nil {

       
    }

    defer resp.Body.Close()


    if resp.StatusCode != http.StatusPartialContent {

        fmt.Println(resp.StatusCode)
        fmt.Println("File server does not support range download.")

    }

    data2, _ := ioutil.ReadAll(resp.Body)
    fmt.Println("before")
   
    data <- &data2

   
    fmt.Println("after")


   

   
}


func getFileSize(url string) (size int64, err error) {
    req, _ := http.NewRequest("GET", url, nil)

    client := &http.Client{}
   
    resp, err1:= client.Do(req)

    if err1 != nil {
        err = err1
        return
    }
    //defer resp.Body.Close()

    fmt.Println(resp.ContentLength)

    return resp.ContentLength, err

}


func genRanges(num int, total int64) (ranges *[]string) {
    single := total / int64(num)
    //fmt.Println(single)

    myranges := make([]string, num)

    for i:=0; i<num-1; i++ {
        myranges[i] = strconv.FormatInt(int64(i)*single, 10) + "-" +  strconv.FormatInt(int64(i+1)*single-1, 10)
        //fmt.Println(myranges[i])
    }
    myranges[num-1] = strconv.FormatInt(int64(num-1)*single, 10) + "-"

    return &myranges

}






func main() {
    url := "http://localhost:8080/Test.zip"

    num := 10
    total, _  := getFileSize(url)

    myranges := genRanges(num, total)
    single := total / int64(num)

    data := make([][]byte, num)

    // fmt.Println(*myranges)
    for i, ran := range *myranges {
        var mydata chan *[]byte
       
        data[i] = make([]byte, single)
        var err error = nil

        go downRange(ran, url, mydata)

        mynewdata := <- mydata
        copy(data[i], *mynewdata)


        fmt.Println(data[i])
        if err != nil {
            fmt.Println(err)
        }
    }

    perm := os.ModePerm

    fulldata := bytes.Join(data, nil)
    ioutil.WriteFile("C:/a.zip", fulldata, perm)
    fmt.Println("Download successful.")   

}

Running Results:
C:\works\go>go run download2.go
7360368
Downloading range: 0-736035
before
_

It stuck at right after before.  I have no idea what happened,  maybe I still does not understand how it works for go routine and channel.
Could anyone help me explain a little bit on the above issue,  why it stuck at right after "before" was printed.
Thank you in advance.

 

[Shawn Milochik]

The execution of the program will end when main() completes. Writing to stdout from goroutines isn't guaranteed to actually make it to the screen.  If you need to make sure all goroutines complete before ending the program, one way is to have each of them write to a channel when done, and have main() wait until it gets a response per goroutine.

As a quick test, use time.Sleep to sleep for 30 seconds at the end of main. If your missing output magically appears, then it's probably a race condition with goroutines being killed when main ends.

[Rui Ueyama]

"mydata" is a nil channel, and nil channel is never ready for communication. A send or a receive on a nil channel blocks forever. You want to change this line to something like this.

mydata := make(chan *[]byte, 1)

        
        data[i] = make([]byte, single)
        var err error = nil

        go downRange(ran, url, mydata)

        mynewdata := <- mydata
        copy(data[i], *mynewdata)


        fmt.Println(data[i])
        if err != nil {
            fmt.Println(err)
        }
    }

    perm := os.ModePerm

    fulldata := bytes.Join(data, nil)
    ioutil.WriteFile("C:/a.zip", fulldata, perm)
    fmt.Println("Download successful.")   

}

Running Results:
C:\works\go>go run download2.go
7360368
Downloading range: 0-736035
before
_

It stuck at right after before.  I have no idea what happened,  maybe I still does not understand how it works for go routine and channel.
Could anyone help me explain a little bit on the above issue,  why it stuck at right after "before" was printed.
Thank you in advance.

 

--
You received this message because you are subscribed to the Google Groups "golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[Wei Ding]

Thank you Shawn.  I wasn't aware of that main routine may not wait for all go routine to complete before it ends. 

在 2014年7月4日星期五UTC+8上午9时56分14秒，Shawn Milochik写道：

[Wei Ding]

Thank you Rui.   Perfect shot!  Yes,  this is the issue.  After I allocate first, it can run without stuck.

Another issue is:
However there might be some issues with my algorithm of downloading. I cannot open the zip file after download.
Is there any order issue here when using the channel,  it seems the file partial bodies were not copied in sequence so that the zip file cannot be opened?
Any suggestions on how to deal with this,  I still wants to use go routine to download concurrently.
Thank you all.


Running results:   (Removed some debugging comments)

C:\works\go>go run download2.go
7360368
Downloading range: 0-736035

Downloading range: 736036-1472071
Downloading range: 1472072-2208107
Downloading range: 2208108-2944143
Downloading range: 2944144-3680179
Downloading range: 3680180-4416215
Downloading range: 4416216-5152251
Downloading range: 5152252-5888287
Downloading range: 5888288-6624323
Downloading range: 6624324-
Download successful.



在 2014年7月4日星期五UTC+8上午9时59分25秒，Rui Ueyama写道：

[Rui Ueyama]

I can't really spend time to understand and debug your code, but if it helps, I'd suggest you run it against a non-ZIP file, perhaps against a text file, so that you can check whether or not the downloaded data actually corrupted with your eyes.

(I don't think parallelizing a download with HTTP range is a good idea. It could rather slow down because of TCP slow start. Also your code does not actually parallelize downloads. But that's a different story as you are writing this code to practice Go programming.)

[Wei Ding]

Thank you Rui for your suggestion.  I've found the problem, which is that the copy function only copy smallest length of src/target slice, which in this case the last piece of range,  the allocated average size might be smaller than actual remaining size.
After changed to below, it works well now.

    for i, ran := range *myranges {

        mydata := make(chan *[]byte, 1)

        data[i] = make([]byte, single)
        var err error = nil

        go downRange(ran, url, mydata)

        mynewdata := <- mydata

        //fmt.Println(*mynewdata)
        //fmt.Println("data[i] length:", len(data[i]))
        //fmt.Println("*mynewdata length: ", len(*mynewdata))
       
        if i != num-1 {
            copy(data[i], *mynewdata)
            //fmt.Println(data[i])

        } else if len(data[i]) < len(*mynewdata) {
           
            tmpdata := make([]byte, len(*mynewdata)-len(data[i]))
            data[i] = append(data[i], tmpdata...)
            copy(data[i], *mynewdata)

        }

        //fmt.Println(data[i])

        if err != nil {
            fmt.Println(err)
        }
    }

If you don't mind,  would you please give me some lights on how to make it concurrently running,   yes,  currently it seems still sequential, as I used go in a for loop.
Thank you very much again.
 



在 2014年7月4日星期五UTC+8上午10时36分51秒，Rui Ueyama写道：

[Rui Ueyama]

Your code is serialized because on each iteration of the loop you spawn a goroutine and wait for it to complete. Only one goroutine is runnable at any moment that way.

To parallelize, make it two pass; spawn goroutines in the first pass, and wait for all of them in the second pass.

[Wei Ding]

Thanks.   I changed to below code, this should be running concurrently right?   How do I determine this from OS perspective, is there any way to check?

package main

import (
    "fmt"
    "os"
    "net/http"
    "io/ioutil"
    "strconv"
    "bytes"

)

func downRange(ran, url string, seq int, data chan map[int] *[]byte) {

   
    fmt.Println("Downloading range: " + ran)
    req, _ := http.NewRequest("GET", url, nil)

    req.Header.Add("Range", "bytes= " + ran)

    // fmt.Println(req.Header)

    client := &http.Client{}
   
    resp, err1:= client.Do(req)

    if err1 != nil {

       
    }

    defer resp.Body.Close()


    if resp.StatusCode != http.StatusPartialContent {

        fmt.Println(resp.StatusCode)
        fmt.Println("File server does not support range download.")

    }

    data2, _ := ioutil.ReadAll(resp.Body)

    //fmt.Println("before")
    data3 := make(map[int] *[]byte, 1)
    data3[seq] = &data2
    data <- data3

   
    //fmt.Println("after")

   

   
}


func getFileSize(url string) (size int64, err error) {
    req, _ := http.NewRequest("GET", url, nil)

    client := &http.Client{}
   
    resp, err1:= client.Do(req)

    if err1 != nil {
        err = err1
        return
    }
    //defer resp.Body.Close()

    fmt.Println(resp.ContentLength)

    return resp.ContentLength, err

}


func genRanges(num int, total int64) (ranges *[]string) {
    single := total / int64(num)
    //fmt.Println(single)

    myranges := make([]string, num)

    for i:=0; i<num-1; i++ {
        myranges[i] = strconv.FormatInt(int64(i)*single, 10) + "-" +  strconv.FormatInt(int64(i+1)*single-1, 10)
        //fmt.Println(myranges[i])
    }
    myranges[num-1] = strconv.FormatInt(int64(num-1)*single, 10) + "-"

    return &myranges

}






func main() {

    url := "http://localhost:8080/Venus.zip"

    num := 10
    total, _  := getFileSize(url)

    myranges := genRanges(num, total)
    single := total / int64(num)

    data := make([][]byte, num)

    flag := -num

    mydata := make(chan map[int] *[]byte, 1)

    for i, ran := range *myranges {
                            

        go downRange(ran, url, i, mydata)


    }

    JLoop:
    for {
       

        select {
            case mynewdata := <- mydata:

                for i, v := range mynewdata {

                   
                    data[i] = make([]byte, single)

                    if i != num-1 {
                        copy(data[i], *v)

                    } else if len(data[i]) < len(*v) {
                       
                        tmpdata := make([]byte, len(*v)-len(data[i]))

                        data[i] = append(data[i], tmpdata...)

                        copy(data[i], *v)


                    }
                    flag++                   
                    fmt.Println("Copied data in range ", i)

                }
            default:
                if flag == 0 {
                    break JLoop

                }
        }

    }




    perm := os.ModePerm

    fulldata := bytes.Join(data, nil)

    ioutil.WriteFile("C:/a1.zip", fulldata, perm)
    fmt.Println("Download successful.")   

}




在 2014年7月4日星期五UTC+8下午1时32分02秒，Rui Ueyama写道：