Help with getting an array of arrays into a 2D array

[Andrew Simper]

When I'm working with time series data I often end up with things like this:

function lowpass(lp::Float64, input::Float64, g::Float64)

    hp::Float64 = input - lp

    lp += g * hp

    [lp, hp]

end

s = 0.0;

data=[flatten([t, lowpass(s, sin(t), 0.5)]) for t in linspace(0,2pi,20)]

20-element Array{Any,1}:
 [0.0,0.0,0.0]                      
 [0.330694,0.16235,0.324699]        
 [0.661388,0.307106,0.614213]       
 [0.992082,0.418583,0.837166]       
 [1.32278,0.4847,0.9694]            
 [1.65347,0.498292,0.996584]        
 [1.98416,0.457887,0.915773]        
 [2.31486,0.367862,0.735724]        
 [2.64555,0.237974,0.475947]        
 [2.97625,0.0822973,0.164595]       
 [3.30694,-0.0822973,-0.164595]     
 [3.63763,-0.237974,-0.475947]      
 [3.96833,-0.367862,-0.735724]      
 [4.29902,-0.457887,-0.915773]      
 [4.62972,-0.498292,-0.996584]      
 [4.96041,-0.4847,-0.9694]          
 [5.2911,-0.418583,-0.837166]       
 [5.6218,-0.307106,-0.614213]       
 [5.95249,-0.16235,-0.324699]       
 [6.28319,-1.22465e-16,-2.44929e-16]

Can anyone please help out with how to turn an array of arrays like this into a 2D array? (or even to pass data as a single chunk into DataFrames to do the job?)

[Tomas Lycken]

Depending on the number of samples in your actual problem this might not be very performant (because ... is slow for large arrays) but here’s an approach using vcat. I had to generate the data slightly differently from you, because I had no function flatten.

julia> data = [[t,lowpass(s,sin(t),0.5)...] for t in linspace(0,2pi,20)]
 20-element Array{Any,1}:   [0.0,0.0,0.0]
  # etc...
julia> vcat([d' for d in data]...)
 20x3 Array{Float64,2}:
  0.0        0.0           0.0
  0.330694   0.16235       0.324699
  # etc...

[Johan Sigfrids]

I suspect it is easier to just pre-allocate an array of the correct dimensions and then assign into it. Something like this:

function lowpassarray(arr::Vector{Float64})
    out = Array(Float64, length(arr), 3)
    s = 0.0
    for i in 1:length(arr)
        out[i, 1] = arr[i]
        out[i, 2], out[i, 3] = lowpass(s, sin(arr[i]), 0.5)
    end
    out
end

data = linspace(0, 2pi, 20)
data = lowpassarray(data, 0.0)

20x3 Array{Float64,2}:
 0.0        0.0           0.0        
 0.330694   0.16235       0.324699   
 0.661388   0.307106      0.614213   
 0.992082   0.418583      0.837166   
 1.32278    0.4847        0.9694     
 1.65347    0.498292      0.996584   
 1.98416    0.457887      0.915773   
 2.31486    0.367862      0.735724   
 2.64555    0.237974      0.475947   
 2.97625    0.0822973     0.164595   
 3.30694   -0.0822973    -0.164595   
 3.63763   -0.237974     -0.475947   
 3.96833   -0.367862     -0.735724   
 4.29902   -0.457887     -0.915773   
 4.62972   -0.498292     -0.996584   
 4.96041   -0.4847       -0.9694     
 5.2911    -0.418583     -0.837166   
 5.6218    -0.307106     -0.614213   
 5.95249   -0.16235      -0.324699   
 6.28319   -1.22465e-16  -2.44929e-16

[Andrew Simper]

Hi Johan,

Thanks for posting that example, that really helped to speed things up.

Pre-allocating the array and storing the computation directly into that array is what I was hoping the array comprehension could do, but I can't trick it into inserting the array directly into a 2D array

I meant to have the first entry be the input signal not t alone. I also want a syntax that will keep all the important information in a single line that is easy to understand what the input signal is and what arguments are being passed to the number crunching function, so I came up with this alternative:

function lowpass(input::Float64, lp::Float64, g::Float64)

    hp::Float64 = input - lp

    lp += g * hp

    [lp, hp]

end

function processarray(f, input::Vector{Float64}, args...)

    ret = f(input[1], args...)

    output = Array(Float64, length(input), length(ret)+1)

    output[1,:] = [input[1], ret]

    for i in 1:length(input)-1

        ret = f(input[i], args...)

        output[i,:] = [input[i], ret]

    end

    output

end

s = 0.0;

data = processarray(lowpass, [sin(t) for t in linspace(0,2pi,2*44100)], s, 0.5)

So the last line here you can see the function lowpass being called with the input of sin(t) and arguments s, and 0.5, so not quite as nice as the array comprehension syntax, but not too bad.

I would love to see support for parametric multi-dimensional array comprehensions, since then I could write something like this, which is really powerful:

s = 0.0;

data = [begin input=sin(t); [t, input, lowpass(input, s, 0.5)]... end for t in linspace(0,2pi,2*44100)]

88200x4 Array{Float64,2}:

 0.0          0.0           0.0            0.0                                

 7.12387e-5   7.12387e-5    3.56194e-5     7.12387e-5    

 0.000142477  0.000142477   7.12387e-5     0.000142477 

 0.000213716  0.000213716   0.000106858    0.000213716

 0.000284955  0.000284955   0.000142477    0.000284955

 0.000356194  0.000356194   0.000178097    0.000356194

 0.000427432  0.000427432   0.000213716    0.000427432

 0.000498671  0.000498671   0.000249336    0.000498671

 0.00056991   0.00056991    0.000284955    0.00056991   

 0.000641149  0.000641149   0.000320574    0.000641149

 0.000712387  0.000712387   0.000356194    0.000712387

 0.000783626  0.000783626   0.000391813    0.000783626

 0.000854865  0.000854865   0.000427432    0.000854865

 ⋮                                                   

 6.2824      -0.000783626  -0.000391813  -0.000783626  

 6.28247     -0.000712387  -0.000356194  -0.000712387 

 6.28254     -0.000641149  -0.000320574  -0.000641149 

 6.28262     -0.00056991   -0.000284955  -0.00056991   

 6.28269     -0.000498671  -0.000249336  -0.000498671 

 6.28276     -0.000427432  -0.000213716  -0.000427432 

 6.28283     -0.000356194  -0.000178097  -0.000356194 

 6.2829      -0.000284955  -0.000142477  -0.000284955  

 6.28297     -0.000213716  -0.000106858  -0.000213716 

 6.28304     -0.000142477  -7.12387e-5   -0.000142477  

 6.28311     -7.12387e-5   -3.56194e-5   -7.12387e-5    

 6.28319     -2.44929e-16  -1.22465e-16  -2.44929e-16