handling large data structure
ld
loaded from CSV-like files which typically contain +100000 records,
with several files at once. The process-flow looks like:
- read a line (lazily)
- split the line into list of ByteString using a separator (say \t)
- convert this list into the (large) data structure
- compute various things using the data structure
- convert the resulting (large) data structure into a list of
ByteString
- save the list using a separator (say \t)
The data structure are generally immutable and used as a lazy storage
for lookup, so I found labelled fields convenient to deal with it. The
small program hereafter show a simple case with a medium-size data
structure and I would like to know if the approach is correct before
going on with other data structure or if there is a more Haskell-ish
way to do this. One more thing, the next step will be to encapsulate
most first-level fields (RefParm) by the Maybe monad since most of
them are optional but their columns are present in the file (empty or
NULL column).
Thanks.
ld.
import Data.ByteString.Char8 (ByteString, pack, unpack)
data RefParm = RefParm {
item :: ByteString, -- pkey
aperture :: Int, -- pkey
harmonic :: ByteString, -- pkey
cside :: Bool,
irng :: IRng,
beamScreen :: BmScr,
geo :: Geo,
dcmag :: DCMag,
sat1 :: Sat,
sat2 :: Sat,
resmag :: ResMag,
penfld :: PenFld,
decay :: Decay,
sbk :: SBk
} deriving (Read, Show)
data IRng = IRng {
ir_item :: ByteString,
ir_low :: Double,
ir_high :: Double
} deriving (Read, Show, Eq)
data BmScr = BmScr {
bs_item :: ByteString,
bs_gamma :: Double
} deriving (Read, Show, Eq)
data Geo = Geo {
geo_item :: ByteString,
geo_gamma :: Double,
geo_i_ref :: Double
} deriving (Read, Show, Eq)
data DCMag = DCMag {
dm_item :: ByteString,
dm_mu :: Double,
dm_p :: Double,
dm_q :: Double,
dm_h :: Double,
dm_t_c0 :: Double,
dm_t_meas :: Double,
dm_i_c :: Double,
dm_i_inj :: Double
} deriving (Read, Show, Eq)
data Sat = Sat {
sat_item :: ByteString,
sat_sigma :: Double,
sat_i0 :: Double,
sat_s :: Double,
sat_i_nom :: Double
} deriving (Read, Show, Eq)
data ResMag = ResMag {
rm_item :: ByteString,
rm_rho :: Double,
rm_r :: Double,
rm_i_inj :: Double
} deriving (Read, Show, Eq)
data PenFld = PenFld {
pf_item :: ByteString,
pf_a :: Double,
pf_alpha :: Double
} deriving (Read, Show, Eq)
data Decay = Decay {
dec_item :: ByteString,
dec_tau :: Double,
dec_d :: Double,
dec_delta :: Double,
dec_i_inj :: Double,
dec_b_inj :: Double
} deriving (Read, Show, Eq)
data SBk = SBk {
sbk_item :: ByteString,
sbk_g :: Double,
sbk_i_inj :: Double
} deriving (Read, Show, Eq)
instance Eq RefParm where
x == y = eq item x y &&
eq aperture x y &&
eq harmonic x y
where
eq f x y = (f x) == (f y)
instance Ord RefParm where
x <= y
| lt item x y ||
eq item x y && lt aperture x y ||
eq item x y && eq aperture x y && lt harmonic x y = True
| otherwise = False
where
eq f x y = (f x) == (f y)
lt f x y = (f x) <= (f y)
fromList :: [ByteString] -> RefParm
fromList xs = RefParm {
item = s 0,
aperture = i 1,
harmonic = s 2,
cside = b 3,
irng = IRng (s 4) (d 5) (d 6),
beamScreen = BmScr (s 7) (d 8),
geo = Geo (s 9) (d 10) (d 11),
dcmag = DCMag (s 12) (d 13) (d 14) (d 15) (d 16) (d 17) (d
18) (d 19) (d 20),
sat1 = Sat (s 21) (d 22) (d 23) (d 24) (d 25),
sat2 = Sat (s 26) (d 27) (d 28) (d 29) (d 30),
resmag = ResMag (s 31) (d 32) (d 33) (d 34),
penfld = PenFld (s 35) (d 36) (d 37),
decay = Decay (s 38) (d 39) (d 40) (d 41) (d 42) (d 43),
sbk = SBk (s 44) (d 45) (d 46)
}
where
s n = xs !! n
b n = read $ unpack (xs !! n) :: Bool
i n = read $ unpack (xs !! n) :: Int
d n = read $ unpack (xs !! n) :: Double
toList :: RefParm -> [ByteString]
toList p = [
item p,
s $ aperture p,
harmonic p,
s $ cside p] ++
(l1 $ irng p) ++
(l2 $ beamScreen p) ++
(l3 $ geo p) ++
(l4 $ dcmag p) ++
(l5 $ sat1 p) ++
(l5 $ sat2 p) ++
(l6 $ resmag p) ++
(l7 $ penfld p) ++
(l8 $ decay p) ++
(l9 $ sbk p)
where
s x = pack $ show x
l1 (IRng i x y) = [ i, s x, s y ]
l2 (BmScr i x) = [ i, s x ]
l3 (Geo i x y) = [ i, s x, s y ]
l4 (DCMag i x y z p q r u v) = [ i, s x, s y, s z, s p, s q, s
r, s u, s v ]
l5 (Sat i x y z v) = [ i, s x, s y, s z, s v ]
l6 (ResMag i x y z) = [ i, s x, s y, s z ]
l7 (PenFld i x y) = [ i, s x, s y ]
l8 (Decay i x y z p q) = [ i, s x, s y, s z, s p, s q ]
l9 (SBk i x y) = [ i, s x, s y ]
--------------------
-- validity tests --
--------------------
main = do
print $ map toList defaultRefParms
print $ map fromList defaultRefParmStrs
print $ (defaultRefParms == (map fromList defaultRefParmStrs))
print $ (defaultRefParmStrs == (map toList defaultRefParms))
print $ (defaultRefParms !! 0) <= (defaultRefParms !! 1)
defaultRefParmStrs = map (map pack) [
[ "HCMBXX", "1", "TF", "True",
"GM", "0.0", "0.0",
"GM", "0.0",
"GM", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0" ],
[ "HCMBXX", "2", "TF", "True",
"GM", "0.0", "0.0",
"GM", "0.0",
"GM", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0",
"GM", "0.0", "0.0", "0.0", "0.0", "0.0",
"GM", "0.0", "0.0" ]
]
defaultRefParms = [
RefParm {
item = pack "HCMBXX",
aperture = 1,
harmonic = pack "TF",
cside = True,
irng = IRng (pack "GM") 0 0,
beamScreen = BmScr (pack "GM") 0,
geo = Geo (pack "GM") 0 0,
dcmag = DCMag (pack "GM") 0 0 0 0 0 0 0 0,
sat1 = Sat (pack "GM") 0 0 0 0,
sat2 = Sat (pack "GM") 0 0 0 0,
resmag = ResMag (pack "GM") 0 0 0,
penfld = PenFld (pack "GM") 0 0,
decay = Decay (pack "GM") 0 0 0 0 0,
sbk = SBk (pack "GM") 0 0
},
RefParm {
item = pack "HCMBXX",
aperture = 2,
harmonic = pack "TF",
cside = True,
irng = IRng (pack "GM") 0 0,
beamScreen = BmScr (pack "GM") 0,
geo = Geo (pack "GM") 0 0,
dcmag = DCMag (pack "GM") 0 0 0 0 0 0 0 0,
sat1 = Sat (pack "GM") 0 0 0 0,
sat2 = Sat (pack "GM") 0 0 0 0,
resmag = ResMag (pack "GM") 0 0 0,
penfld = PenFld (pack "GM") 0 0,
decay = Decay (pack "GM") 0 0 0 0 0,
sbk = SBk (pack "GM") 0 0
}
]
Dirk Thierbach
> I am trying to deal in a convenient way with large data structure
> loaded from CSV-like files which typically contain +100000 records,
> with several files at once. The process-flow looks like:
>
> - read a line (lazily)
> - split the line into list of ByteString using a separator (say \t)
> - convert this list into the (large) data structure
> - compute various things using the data structure
> - convert the resulting (large) data structure into a list of
> ByteString
> - save the list using a separator (say \t)
>
> The data structure are generally immutable and used as a lazy storage
> for lookup,
So if I understand that correctly, the size comes from the number of
records, and not from the fact that one record by itself is particularly
large. Is that correct?
If so, in what kind of structure do you want to store all the records?
What key(s) are you using for lookup? That's probably crucial for performance.
> so I found labelled fields convenient to deal with it. The
> small program hereafter show a simple case with a medium-size data
> structure and I would like to know if the approach is correct before
> going on with other data structure or if there is a more Haskell-ish
> way to do this.
It doesn't hurt to unbox fields, that will remove one level of indirection
and so shrink memory requirements.
Also, if you split the input line with ByteString.split and then just
use some fields in your records as a whole, I suspect that the whole
line won't be garbage collected, because the fields refer to part of
the original line, which is probably allocated as one big chunk. So
one should use ByteString.copy for those fields.
> One more thing, the next step will be to encapsulate most
> first-level fields (RefParm) by the Maybe monad since most of them
> are optional but their columns are present in the file (empty or
> NULL column).
That's sound, but will use comparatively much space per field, which
can hurt if you do have a lot of records. It might be better to add
an extra "flag" field and use Data.Bits to store the presence/absence
information. Then, use "smart" accessors/constructors to wrap
conversion to/from Maybe.
> fromList :: [ByteString] -> RefParm
> fromList xs = RefParm {
> item = s 0,
> aperture = i 1,
> harmonic = s 2,
> cside = b 3,
> irng = IRng (s 4) (d 5) (d 6),
> beamScreen = BmScr (s 7) (d 8),
> geo = Geo (s 9) (d 10) (d 11),
> dcmag = DCMag (s 12) (d 13) (d 14) (d 15) (d 16) (d 17) (d
> 18) (d 19) (d 20),
> sat1 = Sat (s 21) (d 22) (d 23) (d 24) (d 25),
> sat2 = Sat (s 26) (d 27) (d 28) (d 29) (d 30),
> resmag = ResMag (s 31) (d 32) (d 33) (d 34),
> penfld = PenFld (s 35) (d 36) (d 37),
> decay = Decay (s 38) (d 39) (d 40) (d 41) (d 42) (d 43),
> sbk = SBk (s 44) (d 45) (d 46)
> }
> where
> s n = xs !! n
> b n = read $ unpack (xs !! n) :: Bool
> i n = read $ unpack (xs !! n) :: Int
> d n = read $ unpack (xs !! n) :: Double
The list index operator (!!) is slow (it has to walk the complete list
upto the element), which will hurt performance if you do that 100000
times. Use pattern matching instead.
List append (++) is also slow. Better use cons (:) with a continuation, e.g.
something like (untested):
toList :: RefParm -> [ByteString]
toList p =
(item p)
: s (aperture p)
: (harmonic p)
: s (cside p)
: l1 (irng p)
$ l2 (beamScreen p)
$ l3 (geo p)
$ l4 (dcmag p)
$ l5 (sat1 p)
$ l5 (sat2 p)
$ l6 (resmag p)
$ l7 (penfld p)
$ l8 (decay p)
$ l9 (sbk p)
$ []
where
s x = pack $ show x
l1 (IRng i x y) k = i : s x : s y : k
l2 (BmScr i x) k = i : s x : k
....
That's all I can think of for now. All ideas untested.
HTH,
- Dirk
ld
wrote:
> ld <Laurent.Den...@gmail.com> wrote:
> > I am trying to deal in a convenient way with large data structure
> > loaded from CSV-like files which typically contain +100000 records,
> > with several files at once. The process-flow looks like:
>
> > - read a line (lazily)
> > - split the line into list of ByteString using a separator (say \t)
> > - convert this list into the (large) data structure
> > - compute various things using the data structure
> > - convert the resulting (large) data structure into a list of
> > ByteString
> > - save the list using a separator (say \t)
>
> > The data structure are generally immutable and used as a lazy storage
> > for lookup,
>
> So if I understand that correctly, the size comes from the number of
> records, and not from the fact that one record by itself is particularly
> large. Is that correct?
Well, it might come from the product of the two ;-)
A single record contains typically from 25 - 75 columns of Excel-like
sheets which could be either Int, Double, Date, String or empty/NULL.
So I expect something like 1Kb per record in average and therefore
about 100MB for the all set. But most analysis will discard about 90%
of the record through filtering before starting data extraction/use
from the records.
> If so, in what kind of structure do you want to store all the records?
> What key(s) are you using for lookup? That's probably crucial for performance.
The key are the first 3 fields of RefParm (marked as pkey). The
records will be stored in a list, but some of them (with a special
status) will be in a Data.Map for future lookup. The key will be a
RefParmKey which contain these 3 fields.
> > so I found labelled fields convenient to deal with it. The
> > small program hereafter show a simple case with a medium-size data
> > structure and I would like to know if the approach is correct before
> > going on with other data structure or if there is a more Haskell-ish
> > way to do this.
>
> It doesn't hurt to unbox fields, that will remove one level of indirection
> and so shrink memory requirements.
Yes, but these data will be collected into vector (i.e. columns) for
analysis and therefore shared in many places. About unboxed values,
they will also be taken many time since they represent model
parameters to be evaluated many times. So the cost of boxing them each
time they are used may be important.
I also would like to keep the code clean and simple as much as
possible and improve/complexify it only after profiling. I am not yet
sure how laziness will behave with these large records (strict in
construction but lazy in evaluation, right?).
> Also, if you split the input line with ByteString.split and then just
> use some fields in your records as a whole, I suspect that the whole
> line won't be garbage collected, because the fields refer to part of
> the original line, which is probably allocated as one big chunk. So
> one should use ByteString.copy for those fields.
Good point.
> > One more thing, the next step will be to encapsulate most
> > first-level fields (RefParm) by the Maybe monad since most of them
> > are optional but their columns are present in the file (empty or
> > NULL column).
>
> That's sound, but will use comparatively much space per field, which
> can hurt if you do have a lot of records. It might be better to add
> an extra "flag" field and use Data.Bits to store the presence/absence
> information. Then, use "smart" accessors/constructors to wrap
> conversion to/from Maybe.
For the moment I use special values. I will see if Maybe (or Either)
could help because the same remark can be done as for unboxed values.
Can you give me an example? I don't see how to do it without building
RefParm incrementally which in turn will create many temporary
instances of RefParm (unless GHC optimized it). But it's true that it
would be nice to be able to report the column number if 'read' fails.
BTW, I don't know how to catch parsing errors from 'read' without
using Parsec first to ensure proper format or to rely on readIO (and
it's not the right place for it).
Good point. The sequence of $ must be in () because of the precedence
of :, but otherwise it works fine.
> That's all I can think of for now. All ideas untested.
First tests show that it reads, converts from and to list and writes
about 15000 records / second on my laptop. This should be enough for
the moment. I was expecting something much slower considering the
number of read $ unpack and !! performed. Ok for the moment I do
nothing with the records (only one in memory at a time) but the heap
stats are good:
602,473,908 bytes allocated in the heap
3,952,872 bytes copied during GC
36,368 bytes maximum residency (1 sample(s))
43,276 bytes maximum slop
1 MB total memory in use (0 MB lost due to
fragmentation)
Generation 0: 1125 collections, 0 parallel, 0.02s, 0.02s
elapsed
Generation 1: 1 collections, 0 parallel, 0.00s, 0.00s
elapsed
INIT time 0.00s ( 0.00s elapsed)
MUT time 0.99s ( 1.13s elapsed)
GC time 0.02s ( 0.03s elapsed)
EXIT time 0.00s ( 0.00s elapsed)
Total time 1.02s ( 1.16s elapsed)
%GC time 2.1% (2.2% elapsed)
Alloc rate 605,306,743 bytes per MUT second
Productivity 97.9% of total user, 86.1% of total elapsed
Thanks for your comments.
Regards,
ld.
Dirk Thierbach
> The key are the first 3 fields of RefParm (marked as pkey). The
> records will be stored in a list, but some of them (with a special
> status) will be in a Data.Map for future lookup.
Ok, then I think you need to be careful in processing the huge list.
>> It doesn't hurt to unbox fields, that will remove one level of indirection
>> and so shrink memory requirements.
> Yes, but these data will be collected into vector (i.e. columns) for
> analysis and therefore shared in many places.
I'm not sure I understand that. How exactly is sharing introduced?
From the description, you build the data structure by lines, which
means that identical data will end up unshared. Or do you mean
that the analysis will produce new subrecords, and that these should
be shared?
Then in any case I'd unbox the fields with basic types (Int, Double, etc.)
> About unboxed values, they will also be taken many time since they
> represent model parameters to be evaluated many times. So the cost
> of boxing them each time they are used may be important.
If they are already unboxed, strictness analysis should be smart enough
to figure out that they can stay unboxed. In doubt, measure :-)
> I also would like to keep the code clean and simple as much as
> possible and improve/complexify it only after profiling.
Good approach. But adding a few exclamation marks in the type
declarations should be sufficiently localized.
> I am not yet sure how laziness will behave with these large records
> (strict in construction but lazy in evaluation, right?).
Everything is lazy unless either the compiler figures out it can be
strict, you use strictness flags to unbox record fields, or you
explicitely evaluate it. As the thunks created by lazyness usually
take more space than the data itself, it's better if the construction
is strict. Which is automatic if you unbox the fields.
> For the moment I use special values. I will see if Maybe (or Either)
> could help because the same remark can be done as for unboxed values.
Special values are of course bad. If it works fast enough so far, just
use Maybe or Either and worry about optimizing later on, if it's
necessary.
The stupid straightforward way:
fromList [x0,x1,x2,x3, ...etc... ,x46] = RefParm {
item = x0
aperture = i x1
....
where
i x = read $ unpack x :: Int
....
Ugly, but should be faster in case reading is too slow. If reading speed
is ok as it is, just leave it.
> But it's true that it would be nice to be able to report the column
> number if 'read' fails.
Hm, use "reads" in something like (untested)
....
where
....
i n = check n $ reads $ unpack (xs !! n) :: Int
check n [(v,"")] = v
check n _ = error ("Parse error in column " ++ show n)
- Dirk
ld
wrote:
> ld <Laurent.Den...@gmail.com> wrote:
> > The key are the first 3 fields of RefParm (marked as pkey). The
> > records will be stored in a list, but some of them (with a special
> > status) will be in a Data.Map for future lookup.
>
> Ok, then I think you need to be careful in processing the huge list.
>
> >> It doesn't hurt to unbox fields, that will remove one level of indirection
> >> and so shrink memory requirements.
> > Yes, but these data will be collected into vector (i.e. columns) for
> > analysis and therefore shared in many places.
>
> I'm not sure I understand that. How exactly is sharing introduced?
> From the description, you build the data structure by lines, which
> means that identical data will end up unshared. Or do you mean
> that the analysis will produce new subrecords, and that these should
> be shared?
It will produce new sublist and therefore it will share the records.
As I mentioned, >90% of the input will be discarded, then vectors of
data will be extracted from the records and used for calculation. For
example, a list of Geo will be extracted to do average calculation and
other stuffs. So in principle, after few GC cycles, only <10% of the
input should be in memory since the first huge list will be discarded
in the early steps. The point is that I don't know in advance which
records can be discarded before inspecting all of them. In fact, each
'xxx_item' field in sub elements of RefParm (e.g. Geo, DCMag, ..) is a
'soft' link to another RefParm with a matching 'item' name (first
field).
> Then in any case I'd unbox the fields with basic types (Int, Double, etc.)
>
> > About unboxed values, they will also be taken many time since they
> > represent model parameters to be evaluated many times. So the cost
> > of boxing them each time they are used may be important.
>
> If they are already unboxed, strictness analysis should be smart enough
> to figure out that they can stay unboxed. In doubt, measure :-)
> > I also would like to keep the code clean and simple as much as
> > possible and improve/complexify it only after profiling.
>
> Good approach. But adding a few exclamation marks in the type
> declarations should be sufficiently localized.
Good point. I have added exclamation marks to all fields. It doesn't
change much the performance but probably because for the moment I do
almost nothing with the records and GHC sees all the code within a
single module. Nevertheless, I expect a better behavior when the
program will have more modules and analysis.
> > I am not yet sure how laziness will behave with these large records
> > (strict in construction but lazy in evaluation, right?).
>
> Everything is lazy unless either the compiler figures out it can be
> strict, you use strictness flags to unbox record fields, or you
> explicitely evaluate it. As the thunks created by lazyness usually
> take more space than the data itself, it's better if the construction
> is strict. Which is automatic if you unbox the fields.
Well, I tried unboxed fields, but it brings a lot of complication.
Almost all small local functions must be rewritten or explicitly
typed. So I will do it only if I run out of memory (which I doubt).
> > For the moment I use special values. I will see if Maybe (or Either)
> > could help because the same remark can be done as for unboxed values.
>
> Special values are of course bad. If it works fast enough so far, just
> use Maybe or Either and worry about optimizing later on, if it's
> necessary.
Well, the special value here are not 'so bad' because they are
exclusive. The statement looks like <<If xxx_item is not null, that is
S.empty, then use it as (part of) the key to lookup in the Map for
parameter values, otherwise use the following Double values>>.
Ok. I haven't seen a big difference in efficiency (~5%) and it makes
columns error harder to report. I guess that GHC is able to detect the
sequential nature of the list access and optimize it. A bit like in C/C
++ when
for (i = 0; i<n; i++) sum += a[i];
is converted into
for (p = a; p != a+n; p++) sum += *p;
> > But it's true that it would be nice to be able to report the column
> > number if 'read' fails.
>
> Hm, use "reads" in something like (untested)
>
> ....
> where
> ....
> i n = check n $ reads $ unpack (xs !! n) :: Int
> check n [(v,"")] = v
> check n _ = error ("Parse error in column " ++ show n)
Perfect, it works like a charm.
Thanks for all.
a+, ld.
Dirk Thierbach
> Good point. I have added exclamation marks to all fields. It doesn't
[...]
> Well, I tried unboxed fields, but it brings a lot of complication.
Just in case we're talking at cross purposes: By "unboxing record fields"
I mean "adding exclamation marks to the field definitions". Because that
will remove one level of indirection in the representation, hence "unboxing".
That will not so much improve performance, but reduce the memory footprint.
I don't mean "use GHC's unboxed types, i.e. Int#, Double#". That improvement
is often better left to the compiler.
- Dirk
ld
wrote:
> ld <Laurent.Den...@gmail.com> wrote:
> > Good point. I have added exclamation marks to all fields. It doesn't
> [...]
> > Well, I tried unboxed fields, but it brings a lot of complication.
>
> Just in case we're talking at cross purposes: By "unboxing record fields"
> I mean "adding exclamation marks to the field definitions". Because that
> will remove one level of indirection in the representation, hence "unboxing".
>
> That will not so much improve performance, but reduce the memory footprint.
Ok, this is what I did in the end.
> I don't mean "use GHC's unboxed types, i.e. Int#, Double#". That improvement
> is often better left to the compiler.
And this is what I tried with a lot of complication and abandoned when
you told me about unboxing values.
I started to split the all stuff into modules and make it more
generic.
a+, ld.
Mark T.B. Carroll
> I started to split the all stuff into modules and make it more
> generic.
For what it's worth, explicit exports can help optimization by
reassuring the compiler that some of the `internal' functions are only
used in ways that it sees in that module. (Optimization is typically per
module, except perhaps in jhc.) Another bit of help you can give it
there is SPECIALIZE pragmas if it can't see from the module what
concrete types will be used when functions from other modules use it.
Mark
ld
wrote:
> ld <Laurent.Den...@gmail.com> writes:
> > I started to split the all stuff into modules and make it more
> > generic.
>
> For what it's worth, explicit exports can help optimization by
> reassuring the compiler that some of the `internal' functions are only
> used in ways that it sees in that module. (Optimization is typically per
> module, except perhaps in jhc.)
I have read somewhere some time ago that GHC implements cross-module
optimization and inlining.
> Another bit of help you can give it
> there is SPECIALIZE pragmas if it can't see from the module what
> concrete types will be used when functions from other modules use it.
Well, in this particular cases, I export only high level functions to
load/save (lazily) an entire file/list of records. For example, I do
not export fromList and toList but I export:
hLoadRefParm :: Handle -> IO ([ByteString], [RefParm])
hLoadRefParm = T.hLoadTable fromList
hSaveRefParm :: Handle -> ([ByteString], [RefParm]) -> IO ()
hSaveRefParm = T.hSaveTable toList
where T.hLoadTable and T.hSaveTable are in Table module responsible to
deal with IO stuffs for table-like file format (csv, Excel sheet, DB
output, etc...). So the compiler should be smart enough to see that
toList and fromList are never directly used outside the module.
a+, ld.
Mark T.B. Carroll
> I have read somewhere some time ago that GHC implements cross-module
> optimization and inlining.
Oh, cool, now I look at current GHC you seem to be correct. I wonder
when that happened. (-: I wonder what it does with some of my projects
where I build different executables in the same source tree and some of
the common modules between them may be used in different ways by the
different modules associated with particular executables - how eager it
is to recompile an already-compiled module in the face of changes in
other modules.
Mark